{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "week3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dpath = './data2/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGlRJREFUeJzt3X+0XWV95/H3hwBK/ZUgV1ZKoNBO2pHaNmqKUWZaRQuBTgUd7cByJGNZk9oBi2s6XWKnU1TE1rZqa0fp0GUKdKyR+otoY9OUio5WfgRFICBDilYiGYiGnzriAN/54zy3nISbe3fiPjk5yfu11l5n7+9+9j7fs+5Kvmvv59nPTlUhSVIfDhh3ApKkfYdFRZLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqzYHjTmBPO+yww+roo48edxqSNFGuv/76b1XV1Fzt9ruicvTRR7Nhw4ZxpyFJEyXJP3Vp5+0vSVJvRlZUkjw5ybVJvpJkY5K3tvglSb6W5Ia2LGnxJHlvkk1JbkzyvKFzrUhye1tWDMWfn+Smdsx7k2RUv0eSNLdR3v56GDihqh5KchDw+SSfbvt+s6o+skP7k4HFbXkBcBHwgiSHAucDS4ECrk+ypqrubW1WAlcDa4HlwKeRJI3FyK5UauChtnlQW2abZ/9U4LJ23NXA/CQLgZOA9VW1rRWS9cDytu/pVfXFGszffxlw2qh+jyRpbiPtU0kyL8kNwD0MCsM1bdeF7RbXe5I8qcWOAO4cOnxzi80W3zxDfKY8VibZkGTD1q1bf+DfJUma2UiLSlU9WlVLgEXAcUmeA7wZ+JfAzwKHAm9qzWfqD6ndiM+Ux8VVtbSqlk5NzTkiTpK0m/bI6K+qug+4ClheVVvaLa6HgT8HjmvNNgNHDh22CLhrjviiGeKSpDEZ5eivqSTz2/ohwMuAr7a+ENpIrdOAm9sha4Az2yiwZcD9VbUFWAecmGRBkgXAicC6tu/BJMvauc4ErhjV75EkzW2Uo78WApcmmcegeF1eVZ9K8vdJphjcvroBeH1rvxY4BdgEfBd4HUBVbUtyAXBda/e2qtrW1n8NuAQ4hMGoL0d+SdIYZTBwav+xdOnS8ol6ae92/J8cP+4U9nlfeMMXdql9kuuraulc7XyiXpLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2xqEiSemNRkST1xqIiSeqNRUWS1BuLiiSpNxYVSVJvLCqSpN5YVCRJvRlZUUny5CTXJvlKko1J3trixyS5JsntST6c5OAWf1Lb3tT2Hz10rje3+G1JThqKL2+xTUnOG9VvkSR1M8orlYeBE6rqZ4AlwPIky4B3Au+pqsXAvcBZrf1ZwL1V9S+A97R2JDkWOB34SWA58P4k85LMA94HnAwcC5zR2kqSxmRkRaUGHmqbB7WlgBOAj7T4pcBpbf3Utk3b/9IkafHVVfVwVX0N2AQc15ZNVXVHVX0fWN3aSpLGZKR9Ku2K4gbgHmA98I/AfVX1SGuyGTiirR8B3AnQ9t8PPHM4vsMxO4tLksZkpEWlqh6tqiXAIgZXFs+eqVn7zE727Wr8CZKsTLIhyYatW7fOnbgkabfskdFfVXUfcBWwDJif5MC2axFwV1vfDBwJ0PY/A9g2HN/hmJ3FZ/r+i6tqaVUtnZqa6uMnSZJmMMrRX1NJ5rf1Q4CXAbcCnwFe1ZqtAK5o62vaNm3/31dVtfjpbXTYMcBi4FrgOmBxG012MIPO/DWj+j2SpLkdOHeT3bYQuLSN0joAuLyqPpXkFmB1krcDXwY+0Np/APiLJJsYXKGcDlBVG5NcDtwCPAKcXVWPAiQ5B1gHzANWVdXGEf4eSdIcRlZUqupG4LkzxO9g0L+yY/x7wKt3cq4LgQtniK8F1v7AyUqSeuET9ZKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2Zs6gkeUqSA9r6jyd5eZKDRp+aJGnSdLlS+Rzw5CRHAFcCrwMuGWVSkqTJ1KWopKq+C7wS+JOqegWDd8JLkrSdTkUlyQuB1wB/3WKjnDJfkjShuhSVNwJvBj7e3m3yowxetCVJ0nbmvOKoqs8Cn03ylLZ9B/Dro05MkjR5uoz+emF7W+Otbftnkrx/5JlJkiZOl9tffwScBHwboKq+AvzcKJOSJE2mTg8/VtWdO4QeHUEukqQJ12UU151JXgRUkoMZ9KfcOtq0JEmTqMuVyuuBs4EjgM3AkrYtSdJ25iwqVfWtqnpNVR1eVc+qqn9fVd+e67gkRyb5TJJbk2xMcm6LvyXJN5Pc0JZTho55c5JNSW5LctJQfHmLbUpy3lD8mCTXJLk9yYfblZQkaUy6jP66NMn8oe0FSVZ1OPcjwG9U1bOBZcDZSaafxH9PVS1py9p23mOB04GfBJYD708yL8k84H3AyQye5D9j6DzvbOdaDNwLnNUhL0nSiHS5/fXTVXXf9EZV3Qs8d66DqmpLVX2prT/IoB/miFkOORVYXVUPV9XXgE3AcW3ZVFV3VNX3gdXAqUkCnAB8pB1/KXBah98jSRqRLkXlgCQLpjeSHMouTtOS5GgGheiaFjonyY1JVg2d+whgeJTZ5hbbWfyZwH1V9cgOcUnSmHQpKu8C/iHJBUkuAP4B+P2uX5DkqcBHgTdW1QPARcCPMejw39LOD5AZDq/diM+Uw8okG5Js2Lp1a9fUJUm7qEtH/WXAq4C7gXuAV1bVX3Q5eXvvykeBD1bVx9r57q6qR6vqMeDPGNzegsGVxpFDhy8C7pol/i1gfpIDd4jP9BsurqqlVbV0amqqS+qSpN3Q9c2PXwU+BlwBPJTkqLkOaH0eHwBurap3D8UXDjV7BXBzW18DnJ7kSUmOARYD1wLXAYvbSK+DGXTmr6mqYjCx5ava8StafpKkMZmzbyTJG4DzGVypPMrgtlMBPz3HoccDrwVuSnJDi/0Wg9FbS9o5vg78KkCbAfly4BYGI8fOrqpHWw7nAOuAecCqqtrYzvcmYHWStwNfZlDEJElj0qXD/VzgJ7o8mzKsqj7PzP0ea2c55kLgwhnia2c6rs2YfNyOcUnSeHS5/XUncP+oE5EkTb4uVyp3AFcl+Wvg4engcD+JJEnQrah8oy0Ht0WSpBl1efPjWwGSPKWqvjP6lCRJk8o3P0qSeuObHyVJvfHNj5Kk3vjmR0lSb3zzoySpN7NeqbQXZL22ql6zh/KRJE2wWa9U2txbp+6hXCRJE65Ln8oXkvx34MPAPz+nMv1WR0mSpnUpKi9qn28bihWDV/lKkvTP5upTOQC4qKou30P5SJIm2Fx9Ko8B5+yhXCRJE67LkOL1Sf5LkiOTHDq9jDwzSdLE6dKn8ivtc/jZlAJ+tP90JEmTrMssxcfsiUQkSZOvyzvqz5wpXlWX9Z+OJGmSdbn99bND608GXgp8CbCoSJK20+X21xuGt5M8A/iLkWUkSZpYnaa+38F3gcVzNWqjxT6T5NYkG5Oc2+KHJlmf5Pb2uaDFk+S9STYluTHJ84bOtaK1vz3JiqH485Pc1I55b5Lsxu+RJPWky5sfP5lkTVs+BdwGXNHh3I8Av1FVzwaWAWcnORY4D7iyqhYDV7ZtgJMZFKvFwErgovb9hwLnAy8AjgPOny5Erc3KoeOWd8hLkjQiXfpU/nBo/RHgn6pq81wHVdUWYEtbfzDJrQymzz8VeHFrdilwFfCmFr+sqgq4Osn8JAtb2/VVtQ0gyXpgeZKrgKdX1Rdb/DLgNODTHX6TJGkEuhSVbwBbqup7AEkOSXJ0VX2965ckORp4LnANcHgrOFTVliTPas2OAIbfMLm5xWaLb54hLkkaky59Kn8FPDa0/WiLdZLkqcBHgTdW1QOzNZ0hVrsRnymHlUk2JNmwdevWuVKWJO2mLkXlwKr6/vRGWz+4y8mTHMSgoHywqj7Wwne321q0z3tafDNw5NDhi4C75ogvmiH+BFV1cVUtraqlU1NTXVKXJO2GLkVla5KXT28kORX41lwHtZFYHwBurap3D+1aA0yP4FrB453+a4Az2yiwZcD97TbZOuDEJAtaB/2JwLq278Eky9p3nUm3AQSSpBHp0qfyeuCD7UVdMLhCmPEp+x0cD7wWuCnJDS32W8DvAZcnOYtBf82r2761wCnAJgbDll8HUFXbklwAXNfavW260x74NeAS4BAGHfR20kvSGHV5+PEfgWWtbyRV9WCXE1fV55m53wMGT+Xv2L7YftLK4X2rgFUzxDcAz+mSjyRp9Lo8p/KOJPOr6qE2NHhBkrfvieQkSZOlS5/KyVV13/RGVd3L4DaVJEnb6VJU5iV50vRGkkOAJ83SXpK0n+rSUf8/gSuT/DmD50B+hcGT8JIkbadLR/3vJ7kReFkLXVBV60abliRpEnW5UgH4MnAQgyuVL48uHUnSJOsy+uuXgWuBVwG/DFyT5FWjTkySNHm6XKn8V+Bnq+oegCRTwN8BHxllYpKkydNl9NcB0wWl+XbH4yRJ+5kuVyp/k2Qd8KG2/e8YTKkiSdJ2uoz++s0krwT+FYNpVy6uqo+PPDNJ0sTpNPqrTVv/sTkbSpL2a/aNSJJ6Y1GRJPVmp0UlyZXt8517Lh1J0iSbrU9lYZKfB16eZDU7vBulqr400swkSRNntqLyO8B5DN79/u4d9hVwwqiSkiRNpp0Wlar6CPCRJP+tqi7YgzlJkiZUl+dULkjycuDnWuiqqvrUaNOSJE2iLhNK/i5wLnBLW85tMUmSttPl4cdfBJZU1WMASS5lMP39m0eZmCRp8nR9TmX+0PozRpGIJGnydSkqvwt8Ockl7SrleuAdcx2UZFWSe5LcPBR7S5JvJrmhLacM7Xtzkk1Jbkty0lB8eYttSnLeUPyYJNckuT3Jh5Mc3PVHS5JGY86iUlUfApYxmPvrY8ALq2p1h3NfAiyfIf6eqlrSlrUASY4FTgd+sh3z/iTzkswD3gecDBwLnNHaAryznWsxcC9wVoecJEkj1On2V1Vtqao1VXVFVf2fjsd8DtjWMY9TgdVV9XBVfQ3YBBzXlk1VdUdVfR9YDZyaJAyek5l+UdilwGkdv0uSNCLjmPvrnCQ3tttjC1rsCODOoTabW2xn8WcC91XVIzvEZ5RkZZINSTZs3bq1r98hSdrBni4qFwE/BiwBtgDvavHM0LZ2Iz6jqrq4qpZW1dKpqaldy1iS1NmsRSXJAcMd7T+oqrq7qh5tw5P/jMHtLRhcaRw51HQRcNcs8W8B85McuENckjRGsxaV9p//V5Ic1ceXJVk4tPkKYLpgrQFOT/KkJMcAi4FrgeuAxW2k18EMOvPXVFUBnwFe1Y5fAVzRR46SpN3X5eHHhcDGJNcC35kOVtXLZzsoyYeAFwOHJdkMnA+8OMkSBreqvg78ajvXxiSXM3hi/xHg7Kp6tJ3nHGAdMA9YVVUb21e8CVid5O0MHsb8QJcfLEkanS5F5a27c+KqOmOG8E7/46+qC4ELZ4ivBdbOEL+Dx2+fSZL2Al0mlPxskh8BFlfV3yX5IQZXDZIkbafLhJL/kcHzIP+jhY4APjHKpCRJk6nLkOKzgeOBBwCq6nbgWaNMSpI0mboUlYfb0+wAtGG8O30mRJK0/+pSVD6b5LeAQ5L8AvBXwCdHm5YkaRJ1KSrnAVuBmxgMAV4L/PYok5IkTaYuo78ea1PeX8Pgttdt7eFDSZK2M2dRSfKLwJ8C/8hgzq1jkvxqVX161MlJkiZLl4cf3wW8pKo2AST5MeCvAYuKJGk7XfpU7pkuKM0dwD0jykeSNMF2eqWS5JVtdWOStcDlDPpUXs1gokdJkrYz2+2vXxpavxv4+ba+FVjwxOaSpP3dTotKVb1uTyYiSZp8XUZ/HQO8ATh6uP1cU99LkvY/XUZ/fYLBlPWfBB4bbTqSpEnWpah8r6reO/JMJEkTr0tR+eMk5wN/Czw8HayqL40sK0nSROpSVH4KeC1wAo/f/qq2Le2VvvG2nxp3CvuFo37npnGnoL1Ml6LyCuBHh6e/lyRpJl2eqP8KMH/UiUiSJl+XK5XDga8muY7t+1QcUixJ2k6XonL+7pw4ySrg3zCYO+w5LXYo8GEGz7x8Hfjlqro3SYA/Bk4Bvgv8h+mBAElW8Pj7W95eVZe2+POBS4BDGLzj5Vyn5Jek8Zrz9ldVfXampcO5LwGW7xA7D7iyqhYDV7ZtgJOBxW1ZCVwE/1yEzgdeABwHnJ9keoqYi1rb6eN2/C5J0h42Z1FJ8mCSB9ryvSSPJnlgruOq6nPAth3CpwKXtvVLgdOG4pfVwNXA/CQLgZOA9VW1raruBdYDy9u+p1fVF9vVyWVD55IkjUmXNz8+bXg7yWkMrhp2x+FVtaWdd0uSZ7X4EcCdQ+02t9hs8c0zxGeUZCWDqxqOOuqo3UxdkjSXLqO/tlNVn6D/Z1Qy01ftRnxGVXVxVS2tqqVTU1O7maIkaS5dJpR85dDmAcBSZvkPfA53J1nYrlIW8vjLvjYDRw61WwTc1eIv3iF+VYsvmqG9JGmMulyp/NLQchLwIIM+kN2xBljR1lcAVwzFz8zAMuD+dptsHXBikgWtg/5EYF3b92CSZW3k2JlD55IkjUmXPpXdeq9Kkg8xuMo4LMlmBqO4fg+4PMlZwDcYvEUSBkOCTwE2MRhS/Lr23duSXMDjb5p8W1VNd/7/Go8PKf50WyRJYzTb64R/Z5bjqqoumO3EVXXGTna9dKaTAWfv5DyrgFUzxDcAz5ktB0nSnjXblcp3Zog9BTgLeCYwa1GRJO1/Znud8Lum15M8DTiXwW2p1cC7dnacJGn/NWufSnui/T8Dr2HwsOLz2kOIkiQ9wWx9Kn8AvBK4GPipqnpoj2UlSZpIsw0p/g3ghxlM5njX0FQtD3aZpkWStP+ZrU9ll5+2lyTt3ywckqTeWFQkSb2xqEiSemNRkST1xqIiSeqNRUWS1BuLiiSpNxYVSVJvLCqSpN5YVCRJvbGoSJJ6Y1GRJPXGoiJJ6o1FRZLUG4uKJKk3YykqSb6e5KYkNyTZ0GKHJlmf5Pb2uaDFk+S9STYluTHJ84bOs6K1vz3JinH8FknS48Z5pfKSqlpSVUvb9nnAlVW1GLiybQOcDCxuy0rgIhgUIeB84AXAccD504VIkjQee9Ptr1OBS9v6pcBpQ/HLauBqYH6ShcBJwPqq2lZV9wLrgeV7OmlJ0uPGVVQK+Nsk1ydZ2WKHV9UWgPb5rBY/Arhz6NjNLbazuCRpTHb6jvoRO76q7kryLGB9kq/O0jYzxGqW+BNPMChcKwGOOuqoXc1VktTRWK5Uququ9nkP8HEGfSJ3t9tatM97WvPNwJFDhy8C7polPtP3XVxVS6tq6dTUVJ8/RZI0ZI8XlSRPSfK06XXgROBmYA0wPYJrBXBFW18DnNlGgS0D7m+3x9YBJyZZ0DroT2wxSdKYjOP21+HAx5NMf/9fVtXfJLkOuDzJWcA3gFe39muBU4BNwHeB1wFU1bYkFwDXtXZvq6pte+5nSJJ2tMeLSlXdAfzMDPFvAy+dIV7A2Ts51ypgVd85SpJ2z940pFiSNOEsKpKk3oxrSPFEeP5vXjbuFPZ51//BmeNOQVKPvFKRJPXGoiJJ6o1FRZLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2xqEiSemNRkST1xqIiSerNxBeVJMuT3JZkU5Lzxp2PJO3PJrqoJJkHvA84GTgWOCPJsePNSpL2XxNdVIDjgE1VdUdVfR9YDZw65pwkab816UXlCODOoe3NLSZJGoMDx53ADygzxOoJjZKVwMq2+VCS20aa1XgdBnxr3El0lT9cMe4U9iYT9bcD4PyZ/gnutybq75df3+W/3Y90aTTpRWUzcOTQ9iLgrh0bVdXFwMV7KqlxSrKhqpaOOw/tOv92k82/38Ck3/66Dlic5JgkBwOnA2vGnJMk7bcm+kqlqh5Jcg6wDpgHrKqqjWNOS5L2WxNdVACqai2wdtx57EX2i9t8+yj/dpPNvx+Qqif0a0uStFsmvU9FkrQXsajsI5yuZnIlWZXkniQ3jzsX7ZokRyb5TJJbk2xMcu64cxo3b3/tA9p0Nf8b+AUGw6yvA86oqlvGmpg6SfJzwEPAZVX1nHHno+6SLAQWVtWXkjwNuB44bX/+t+eVyr7B6WomWFV9Dtg27jy066pqS1V9qa0/CNzKfj6rh0Vl3+B0NdKYJTkaeC5wzXgzGS+Lyr6h03Q1kkYjyVOBjwJvrKoHxp3POFlU9g2dpquR1L8kBzEoKB+sqo+NO59xs6jsG5yuRhqDJAE+ANxaVe8edz57A4vKPqCqHgGmp6u5Fbjc6WomR5IPAV8EfiLJ5iRnjTsndXY88FrghCQ3tOWUcSc1Tg4pliT1xisVSVJvLCqSpN5YVCRJvbGoSJJ6Y1GRJPXGoiL1KMn8JP9pD3zPi5O8aNTfI+0qi4rUr/lA56KSgd35d/hiwKKivY7PqUg9SjI9Q/RtwGeAnwYWAAcBv11VV7SJBz/d9r8QOA14GfAmBtPr3A48XFXnJJkC/hQ4qn3FG4FvAlcDjwJbgTdU1f/aE79PmotFRepRKxifqqrnJDkQ+KGqeiDJYQwKwWLgR4A7gBdV1dVJfhj4B+B5wIPA3wNfaUXlL4H3V9XnkxwFrKuqZyd5C/BQVf3hnv6N0mwOHHcC0j4swDvaS7geY/A6gsPbvn+qqqvb+nHAZ6tqG0CSvwJ+vO17GXDsYIopAJ7eXgYl7ZUsKtLovAaYAp5fVf8vydeBJ7d93xlqN9OrC6YdALywqv7vcHCoyEh7FTvqpX49CExfSTwDuKcVlJcwuO01k2uBn0+yoN0y+7dD+/6WwWShACRZMsP3SHsNi4rUo6r6NvCFJDcDS4ClSTYwuGr56k6O+SbwDgZvDPw74Bbg/rb719s5bkxyC/D6Fv8k8Io2K+6/HtkPknaRHfXSXiDJU6vqoXal8nFgVVV9fNx5SbvKKxVp7/CWJDcANwNfAz4x5nyk3eKViiSpN16pSJJ6Y1GRJPXGoiJJ6o1FRZLUG4uKJKk3FhVJUm/+P7RyK1qmT6eeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb499a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');\n",
    "#三种类别很不均匀"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#特征工程已完成，直接划分数据集\n",
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#交叉验证参数\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一、训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.初步确定弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHWd//HXp7vnvjKTmdwnSYBwGTFyqrAKcqgcghEUFVxlXRdv18Vdfx7suuKu67XggQisorCoqIgsKoiiciWROxAIue9zMvfV/fn98a2ZdCY9ySRMT81Mv5+PRz2mu6q6+tPVPf3u+lbVt8zdERERAUjEXYCIiIwcCgUREemjUBARkT4KBRER6aNQEBGRPgoFERHpo1AQyWJm/2xmN8Zdh0hcFAqjjJlVmtlqM3tH1rgqM1trZhdnjVtoZneb2S4zazSzZWb2RTOrjaZfbmZpM2uJhpVm9vd5rv10M1ufz+c4GLnqcfd/d/f35en5VpvZGflYdj4M1/s12tbLWKdQGGXcvQW4EviGmTVEo/8DWOLuPwUws1OAPwB/AY5093HA2UAP8IqsxT3s7pXuXglcDPyHmb1yeF6JHAwzS8VdgxQId9cwCgfgFuA24HRgBzA5a9qfgf8+wOMvB/7cb9xjwDuy7p8HPAs0EkJmfta0+dG4xmie87KmnQssA5qBDcAngQqgHcgALdEwZYDXdT3w6+jxjwJzBrE+jgR+B+wElgOLDqUe4PPArdHjZgEOXAGsA3YBHwBeDTwVvfbrsp5nDvD76P3YDvwIGBdN+2H0XO3Rc31qEOt4NfBP0XN1Aqno/obotSwH3pBjXZwEbAaSWeMuBJ6Kbp8ALAGagC3AVwdYp6cD6weYVgP8ANgGrAE+AySiaUngv6J1sAq4KlqPqQGWtRo4Y4Bp7wdWRO/rXb2fGcCArwFbgd3ROjpmoPc77v/X0TTEXoCGQ3zjoBbYFP3jXZE1vgJIA6cf4PGXkxUK0RddI3B4dP9woBU4EygCPhX9cxZH91cA/xzdf330D3hE9NhNwGuz6jw+uj3gl0xWHbdEXwAnRF+CPwJuP8BjKghf2ldEjzk+Wi9HH2w95A6F7wClwBuBDuAXwARgavSldFo0/9xofZUADcCDwNezlr3Xl9/+1nHW/E8A04Ey4IjodU7Jqi9nYAIvAWdm3f8JcHV0+2HgXdHtSuCkAZYx4PtFCIRfAlVRHS8AfxtN+wDhS3latL7v4xBCIfpcbY/ezxLgv4EHo2lnAUuBcYSAmE/0w2ig91vD4AY1H41S7r6L8AuzHLgza1ItoVlwc+8IM/uPaL9Cq5l9Jmvek6LxLYSthB8CL0bT3g782t1/5+7dwFcIX0ynEH6JVgLXunuXu/8euBu4NHpsN3CUmVW7+y53/+tBvrw73f0xd+8hhMKCA8z/ZmC1u9/s7j3R8/2M0CQ2FPX8q7t3uPtvCV/it7n7VnffAPwJeCWAu6+I1lenu28Dvgqctp/l7m8d9/qmu69z93ZC2JdEr6XI3Ve7+0sDLPs2ovfDzKoIv55vy1ofc82s3t1b3P2Rg1kZZpaMav+0uze7+2rClsG7olkWAd9w9/XR5/Tag1l+lncCN7n7X929E/g0cLKZzYpeQxVhC9Hc/Tl335T1+l7O+13QFAqjlJldRviFdh/w5axJuwjNFJN7R7j7pzzsV/g54Zd0r0fcfZyHfQqTgKOBf4+mTSE0C/QuI0P4lTo1mrYuGtdrTTQN4CLCl9AaM/ujmZ18kC9vc9btNkIA7c9M4MQo4BrNrJHwhTJpiOrZknW7Pcf9SgAzm2Bmt5vZBjNrAm4F6vez3P2t417rsqavAD5K2JrZGj3XlAGW/WPgrWZWArwV+Ku79z7X3xK2Up43s8Vm9ub91JhLPWELcU3WuOz3f0p23f1uH4z+66eF0DQ3Nfohch2hqXGLmd1gZtXRrC/3/S5oCoVRyMwmENpT3w/8HbDIzF4H4O6thHb4tx7MMt19C+HX9VuiURsJX7a9z2mEZowN0bTpZpb9+ZkRTcPdF7v7+YQmll8Ad/Q+zcHUdBDWAX+MAq53qHT3vx/mer4ULfM4d68GLiM0bfTq/3z7W8c5H+PuP3b310SPc/b+QZA93zLCF+o5wDsIIdE77UV3v5SwPr4M/NTMKgb/MtlO+DU+M2tc3/tPaL6ZljVt+kEsO1v/9VMBjGfP5+yb7v4qwo+Zw4F/jMYP9H7LICgURqfrgF+4+wPRJvOngO9FvwqJ7r/XzK6OAgQzmwbMHmiBZjaesDPy2WjUHcCbzOwNZlYEfIKws/MhQui0Ap8ysyIzO50QJrebWbGZvdPMaqImkSZCsweEX9jjzaxmiNZDr7uBw83sXVE9RWb2ajObP8z1VBF2Ijea2VSiL6ksW4DDsu7vbx3vw8yOMLPXR+9zB2ErJZ1r3siPgQ8DryPsU+hdzmVm1hBtmTRGowdcjpmVZg+ELdE7gC9Gh0PPBD5O2DLqfV0fMbOpZjaOsHP8QIr6PU8qqv8KM1sQveZ/Bx5199XR+3titN5ao/WRPsD7LYMR904NDQc3ABcQfkGN6zf+fuCLWfdPBO4h/NM3As8AXwTGR9MvJ/yz9B55s5XQ5jwhaxkXEnYY7gb+SLTjNpp2dDRudzTPhdH4YuBeQjNWE7AYeE3W424iNAE0MvDRR/+Wdf90DrBzOprvCMIRS9ui5f+esC/ioOoh947mVNb868naiU/4IvxM1jpZGq3PJwhf8uuz5j0fWBs91ycHsY5Xs/eO6eMI+36aCTvj7861DrPmn0H4Av91v/G3Ru93C+FHwAUDPP706PX3H+YS9l3dGq3vdcBn2XP0UYqwJbuDcPTRxwhbFjbA86zO8Rz/Fk37AGGnee/rnRaNfwPhiKMW9hzpVXmg91vDgQeLVrCISF6Y2TnAd9x95gFnltip+UhEhpSZlZnZuWaWiprRPkc4yEFGAW0pyKhgZq8F/i/XNA9HT8kIYWblhKawIwn7PX4NfMTdm2ItTAZFoSAiIn3UfCQiIn1GXSdb9fX1PmvWrLjLEBEZVZYuXbrd3RsONN+oC4VZs2axZMmSuMsQERlVzGzNgedS85GIiGRRKIiISB+FgoiI9FEoiIhIH4WCiIj0USiIiEgfhYKIiPQpmFBYuWU39zz2LOrWQ0RkYAUTCvd979Oce88ptLa1xl2KiMiIVTChcPz8OQA07dhygDlFRApXwYRCUWW4fnrLrq0xVyIiMnIVTCiU1oR+oNqbtsVciYjIyFUwoVA+LoRC526FgojIQAomFKpqJwLQ07Ij5kpEREauAgqFCQBk2hQKIiIDKZhQSBaX0kopifadcZciIjJiFUwoADRbNcmOXXGXISIyYhVUKLQmqynu3h13GSIiI1ZBhUJ7UQ1l3Y1xlyEiMmIVVCh0FddSmW6KuwwRkRErb6FgZjeZ2VYze2aA6WZm3zSzFWb2lJkdn69aemVKxlHlzfl+GhGRUSufWwq3AGfvZ/o5wLxouBL4dh5rAcDL6qixVto7OvP9VCIio1LeQsHdHwT2d/zn+cAPPHgEGGdmk/NVD0CiYjwAjTvVKZ6ISC5x7lOYCqzLur8+GrcPM7vSzJaY2ZJt2w69m4pUb6d4O9UpnohILnGGguUYl/MKOO5+g7svdPeFDQ0Nh/yEJdUhFNrU/5GISE5xhsJ6YHrW/WnAxnw+Yfm40NVFp3pKFRHJKc5QuAt4d3QU0knAbnfflM8nrIz6P+pu3p7PpxERGbVS+Vqwmd0GnA7Um9l64HNAEYC7fwe4BzgXWAG0AVfkq5Ze1XWhp9RMi0JBRCSXvIWCu196gOkO/EO+nj+XVFkVbZRibQoFEZFcCuqMZoDdVkNRh0JBRCSXgguFlqI6Sjp1TQURkVwKLhQ6iuuo6lH32SIiuRRcKHSX1lOTUU+pIiK5FFwoZCoaqKWJNvV/JCKyj4ILhWRlA0lzdm5T/0ciIv0VXCgU1UwCYPeOvJ48LSIyKhVcKJSPC6HQtjOvJ0+LiIxKBRcKVfWhd+6u3Wo+EhHpr+BCoaY+9M6dblYoiIj0V3ChUFxZRzdJaNU1FURE+iu4UCCRYLdVk2rXWc0iIv0VXigAzUl1dSEikktBhsKmniqK1SmeiMg+CjIUKsZPoYFGQu/dIiLSqyBDIVM5mXoaaW5XVxciItkKMhSS4yaTsgzbt2yIuxQRkRGlIEOhtHYaALu3rI25EhGRkaUgQ6GqYToArTvWx1yJiMjIUpChUDtpJgDduxQKIiLZCjIUSsdNJo1B8+a4SxERGVEKMhRIpmi0WoraFAoiItkKMxSA3UX1lHVsi7sMEZERpWBDob2kgepuhYKISLaCDYXu8kmM9530pDNxlyIiMmIUbChY9WTqrIXtjU1xlyIiMmIUbCj8bn146Ts26wQ2EZFeBRsKi15/AgC7t6yOtxARkREkr6FgZmeb2XIzW2FmV+eYPtPM7jezp8zsD2Y2LZ/1ZKubMgeAjm2rh+spRURGvLyFgpklgeuBc4CjgEvN7Kh+s30F+IG7HwdcA3wpX/X0V9kwC4BM47rhekoRkREvn1sKJwAr3H2lu3cBtwPn95vnKOD+6PYDOabnT3E5jVZDqlk9pYqI9MpnKEwFsn+Gr4/GZXsSuCi6fSFQZWbj+y/IzK40syVmtmTbtqE7t6CxeBKVHZuGbHkiIqNdPkPBcozrf6mzTwKnmdnjwGnABqBnnwe53+DuC919YUNDw5AV2FY2mdruzboCm4hIJJ+hsB6YnnV/GrAxewZ33+jub3X3VwL/Eo3bncea9pKumspkdrC7rWu4nlJEZETLZygsBuaZ2WwzKwYuAe7KnsHM6s2st4ZPAzflsZ59pOpmUm6dbNq88cAzi4gUgLyFgrv3AFcBvwGeA+5w92fN7BozOy+a7XRguZm9AEwEvpivenIpb5gNwK6NLw3n04qIjFipfC7c3e8B7uk37rNZt38K/DSfNexP7ZQQCm06V0FEBCjgM5oBqiYeBkB655qYKxERGRkKOhSsvI52Skk06bKcIiJQ4KGAGTuLJlPRplAQEYFCDwWgtWIa9d0byWR0roKISMGHQk/NLKazlS1N7XGXIiISu4IPheKGwyizLjauXx13KSIisSv4UKiecjgAjRteiLkSEZH4FXwo1E07AoCOrTqBTUSk4EMhVTeTNAls16q4SxERiV3BhwKpYnYmGyhv0bWaRUQUCkBz2TRquzaqC20RKXgKBaC7ZibTfDPbW9SFtogUNoUCUNQwj/HWzKr16kJbRAqbQgGomX4UADvWPBNzJSIi8VIoAHUzjwagY9PzMVciIhIvhQJgtbPoJkVq54txlyIiEiuFAkCyiJ3FU6hqXR13JSIisVIoRFqrDmNqz3paOnviLkVEJDYKhV7185hpm1m5pTHuSkREYqNQiFRMnU+xpfnP238bdykiIrFRKETGzzwGgPOmNsdciYhIfA4YCmY2x8xKotunm9mHzWxc/ksbXqmJRwLgW5fHXImISHwGs6XwMyBtZnOB7wOzgR/ntao4lNawKzWBqiYdlioihWswoZBx9x7gQuDr7v4xYHJ+y4pHc83hzEqvZkdLZ9yliIjEYjCh0G1mlwLvAe6OxhXlr6T42MSjmWMbeGHjzrhLERGJxWBC4QrgZOCL7r7KzGYDt+a3rHiMm3UcxZZm40r1gSQihSl1oBncfRnwYQAzqwWq3P3afBcWh6oZrwCgff3TwJnxFiMiEoPBHH30BzOrNrM64EngZjP76mAWbmZnm9lyM1thZlfnmD7DzB4ws8fN7CkzO/fgX8IQqj+cHhK0rHsq1jJEROIymOajGndvAt4K3OzurwLOONCDzCwJXA+cAxwFXGpmR/Wb7TPAHe7+SuAS4FsHU/yQS5XQWDaTub6Wju50rKWIiMRhMKGQMrPJwCL27GgejBOAFe6+0t27gNuB8/vN40B1dLsGiP0qN931R3GkrWHZpqa4SxERGXaDCYVrgN8AL7n7YjM7DBjMwfxTgXVZ99dH47J9HrjMzNYD9wAfGsRy86pi1quYZttZ/tKquEsRERl2BwwFd/+Jux/n7n8f3V/p7hcNYtmWa3H97l8K3OLu04BzgR+a2T41mdmVZrbEzJZs27ZtEE996KoOezUATauW5PV5RERGosHsaJ5mZj83s61mtsXMfmZm0wax7PXA9Kz709i3eehvgTsA3P1hoBSo778gd7/B3Re6+8KGhoZBPPWhs8nhCKSiLU/m9XlEREaiwTQf3QzcBUwhNP/8Khp3IIuBeWY228yKCTuS7+o3z1rgDQBmNp8QCvndFDiQ0hp2lc5gSttymju6Yy1FRGS4DSYUGtz9ZnfviYZbgAP+XI+6xriKsD/iOcJRRs+a2TVmdl402yeA95vZk8BtwOXu3r+Jadh1TzyOYxKreGr97rhLEREZVgc8eQ3YbmaXEb60IewH2DGYhbv7PYQdyNnjPpt1exlw6uBKHT7Vhy2kdM3d/PrFlzh17j6tWSIiY9ZgthTeSzgcdTOwCbiY0PXFmFU6M+xsbn7p0ZgrEREZXoM5+mitu5/n7g3uPsHdLyCcyDZ2TTmeNEmqtv2VnnQm7mpERIbNoV557eNDWsVIU1xO87gjOTaznOc360psIlI4DjUUcp2DMKakZp3EgsRLLF25Ne5SRESGzaGGQuxHCOVb5ZxTKLdONizXSWwiUjgGPPrIzJrJ/eVvQFneKhoppp8AgG14jEzmEhKJMb9xJCIycCi4e9VwFjLi1EynvXQix7QuY9mmJo6ZWhN3RSIieXeozUdjnxnMeg0nJZ7joRXxnmQtIjJcFAr7UTbvNBpsN6uWPxF3KSIiw0KhsD+zXgNAcs1f6OzRRXdEZOxTKOxP3WF0lE3kxMQyHl25M+5qRETybjBdZzebWVO/YV3UnfZhw1FkbMxImXNyYhkPPLc57mpERPJuMFsKXwX+kdBt9jTgk8D3CJfXvCl/pY0MqTd+gXprYu1zjzECOnAVEcmrwYTC2e7+XXdvdvcmd78BONfd/xeozXN98ZvzegAOb36MldtbYy5GRCS/BhMKGTNbZGaJaFiUNW3s/3SumkRXwzGclnySy296LO5qRETyajCh8E7gXcDWaHgXcJmZlREuojPmFR9xJgsTLzC5tCfuUkRE8mowXWevdPe3uHt9NLzF3Ve4e7u7/3k4iozd3DNIkaZ2y19Yt7Mt7mpERPJmMEcfTYuONNpqZlvM7GdmNm04ihsxpp9IurSWNyaXcM/Tm+KuRkQkbwbTfHQzcBcwhXAE0q+icYUjWUTyiHM4I/E437r/ubirERHJm8GEQoO73+zuPdFwC9CQ57pGnvlvocZaOa7naV7cogvviMjYNJhQ2G5ml5lZMhouA3bku7ARZ87f4EXlnJNczJ2Pb4i7GhGRvBhMKLwXWARsBjYBFwNX5LOoEamoDDv8LN5ctJS7/7qGTGbsH40rIoVnMEcfrXX389y9wd0nuPsFwFuHobaR59i3UZ1p5LCWpTz4orrTFpGx51A7xPv4kFYxWsw9Ay+tYVHJw9z6yNq4qxERGXKHGgqFeW3KVAl21PmcaUt46Pm1bGhsj7siEZEhdaihULgN6sddQnGmjXPsUS7+9kNxVyMiMqQGDIUBusxuMrNmwjkLhWnmKVA3h7+v/gstnT20dKrrCxEZOwYMBXevcvfqHEOVu6eGs8gRxQyOfzdzO55mQucabn9M+xZEZOzI65XXzOxsM1tuZivM7Ooc079mZk9Ewwtm1pjPeobMgncAxodTv+DL9z5PR7cu1SkiY0PeQsHMksD1wDnAUcClZnZU9jzu/jF3X+DuC4D/Bu7MVz1DqnICHHMRbyp9kuJ0G9//86q4KxIRGRL53FI4AVgR9bLaRbhS2/n7mf9S4LY81jO0Tvogqe4WPjP1cb71wAq2NXfGXZGIyMuWz1CYCqzLur8+GrcPM5sJzAZ+P8D0K81siZkt2bZthJw0Nu1VUFzFxY030dnVxZu++ae4KxIRednyGQq5zmUY6FDWS4CfunvOxnl3v8HdF7r7woaGEdQX3wXfoijdxpePfIntLZ28oI7yRGSUy2corAemZ92fBmwcYN5LGE1NR72OfDM0zOeC5ttImHPxtx9Sn0giMqrlMxQWA/PMbLaZFRO++O/qP5OZHQHUAg/nsZb8SCTgdZ8kuWM5P6i5kaaOHn6kQ1RFZBTLWyi4ew/hGs6/AZ4D7nD3Z83sGjM7L2vWS4Hb3X10/sQ++kKYeCwnF6+goTTD5375jC7ZKSKjlo227+KFCxf6kiVL4i5jbyv/AD84n92n/gun/vkVHDO1mh+/7yQSicLsIkpERh4zW+ruCw80X15PXisYh50OZbXUPHQt/3rGRB5ZuZPTv/JA3FWJiBw0hcJQee9vALig6VbOPGoiGxs7WLx6Z8xFiYgcHIXCUGk4AiomYIu/x9dOS5JMGJfe8Airt7fGXZmIyKApFIbSBx+C8noqf/sJ7vnQKVSVpnj3TY+xtbkj7spERAZFoTCUyuvgnC/DhqXMuf00powrY/2uNt5z02KaOrrjrk5E5IAUCkPtmIvgiHOhaQO/XjSOW644gec3NXHyl+5nZ2tX3NWJiOyXQmGomcF5/w043Hgmr5tRzA3vXkhbV5pTrr1fl/AUkRFNoZAPFfVw2c8h0w0/fS9nHlnP7e8/iaJkgou+9ZD6SBKREUuhkC+zXwvjZsKK++B3n+XEw8Zzx9+dzI7WTs75xp9YosNVRWQEUijk04f/CidcCQ9fB0tvYf7kan7/idNJJYy3fedhrvv9i/SkM3FXKSLSR6GQb2d9CeaeAb/6KDx1B9Prynnsn8+grqKYr/z2Bd5+wyOs2aFzGURkZFAo5FsyBYt+CCXVcOf74dmfU1NexNL/dybfuGQBT6xr5G++8gdu/NNK0up2W0Ripg7xhktXK9x6Eax7DBb9D8x/CwCbdrdzztf/RGN7NxUlSWbWlXPPR14Xc7EiMtaoQ7yRprgC3vkTKCqH/30XLL8XgMk1ZTz+2TP55qWvpLM7w7JNzbziC79h6ZpdMRcsIoVIWwrDrWM3fPWosOVw/vXwynf2TWrr6uGsrz3I+l3tOFBTVsSUmlLu+chrMVM33CJy6LSlMFKV1sDHnwt/f/lB+ON/QhTM5cUp/vRPr+eZL5zF1eccSWtnD89tbuboz/2GO5aso70r5yWsRUSGjLYU4tLTBXd9CJ66HSomwEeehOLyvWZp70pz7jceZHNTJ+3dIRAaKou57h3Hc8LsOm09iMigDXZLQaEQJ3f445fhD9fCxGPCDujxc3LM5jy6aidX/fivbG8J/SeVpBLUV5Zw+5UnMb2ufJ/HiIhkUyiMJi/eB3e+DzJpOOvfYcE7IZG7Za+tq4d7n9nM5+96lqaOHgCqSlPUlhczrqyIX151qrYgRGQfCoXRpnEt3HklrH04nNPw3nth4tH7fcgF1/+ZZZuaSZr1NS+VpBKMKy+KAuI1lBYlh6N6ERnhFAqjUSYDT/wI7v4oZHrglA/Daf8EJZUHfOi6nW286/uP0tjWTVNHN73nwVWXpqguK6KmNGxFpJI6tkCkECkURrPWHXDf5+DxH0KyBC76Hsw/L3TLPQgd3WnOv+7PrNzeSlEyQVt01FLCoKIkRUd3mpl15dx0+QlMrytTc5NIAVAojAVrH4EfXgjdbTDtBDjj8zDr1INezI6WTt7+3Ydp7uyhpbOH1s49h7YWJQ13mFxTSkVJivLiJHd+8OCfQ0RGNoXCWJHugSduhXv+EdJdUFYL77kbJh1zyIvsTme48Pq/0BKFxI7WLrI/BgbUlBdRXpRkZ1sXcxsqueMDJ1NenHr5r0dEYqFQGGu62uCx78Lv/zUcpXTc28P+hhyHsB6KxrYuLrnhEVZtbyWdcVJJo7M7Q/anwwwSZjRUFlNalGRLUwdHTqrmzg+eoiYokRFOoTBWte+CP38dHvomeAbKxsM7bofpJwz5U3WnM1z0rYdo707T3pVmc1MHmejzkt2ha8KgtChJV0+GidWllBUl2LC7g3kTKvnR+06ksiSl0BCJmUJhrGvZCo9+F/7y9XCk0vST4JQPwRHnDniOw1Bxdy781kMs39xExmFceREd3Wma2nvI9WkyAxzKS5IUJRO0dvZgwORxZRQlExQljOvfeTz1VSVUKUBE8kKhUCg6W+DxW+F3n4V0J6RK4fX/Dxa8A8rrhr2cju40b/vOQ6zY2oIDDZUldKed7S2dlBcn6ck47V3pnOHRy4Dy4iRFqawAqSmjKGlhC6Shkh++70SqSxUgIoM1IkLBzM4GvgEkgRvd/doc8ywCPg848KS7v2N/y1QoDCDdA8t+AY/dAOseBQxecQksfC9Me/WgD2cdLou+8xA9Gac7nWHF1hamjCujO51h0+4O3KGiJEl3evABkkwYbV1pasuLSSaMnW1dGDB1XBmphLGusZ25DRUkzVixrYWjJldzxwdOGaZXKxK/2EPBzJLAC8CZwHpgMXCpuy/LmmcecAfwenffZWYT3H3r/parUBiEzc/A0pthyc3g6dCv0sIr4NhFUFodd3UH7cABkiKdcdq6ekgmjHTGGcxF7JIJI5kwetIZDKgsLSKZMJo7uqmrKCZpxvbWEC7Ta8tIJoy1u0K4JKJwMVDAyKgwEkLhZODz7n5WdP/TAO7+pax5/gN4wd1vHOxyFQoHobMFnv4J/PZfwvUbLAHz3ghHnBP2PVROiLvCvOlOZ3j7dx8mnXHSGeelbS1Mqy2nJ+NsbGzH3amtKCadcRrbunGgNJUg7U5nd4aEGemD+N9IJoxMxikpSpBMGB3dIWhqyopImNHY3kVDZQmJhLGtuRMIWzFmsKGxHQNmjq8gYbB6RxtzouAxg++/59UUpxKUpJIUJU1NZnJIRkIoXAyc7e7vi+6/CzjR3a/KmucXhK2JUwlNTJ9393tzLOtK4EqAGTNmvGrNmjV5qXnMcocNS0NALLk57HsAmPVaOPqCcLb0GA6IQ7XoOw+xbFMTDhxWX0k6k2HV9lam1ZaTzjgbGsPFkOorikm7s6O1i+rSItIZp6UjBE1RMkHGne700P2fGaE1MOPhdlEygVkIwrKiJAkz2rrTWaFYUp8GAAAQcUlEQVQEje3dAEyoKiFhxtbmTibXlGJmbNrdDsCM2nLMYO3ONmbXV/Dli15BUcooTiYoTiX6/hYlQ/D1/pXRYSSEwtuAs/qFwgnu/qGsee4GuoFFwDTgT8Ax7t440HK1pfAyucOWZ+G2S6B1O/SELwRmvw6OvjAEREV9vDWOUe5OZ0+Gzp4MV9z8GJmomWvFthYAZtaVk3Fn7c42ptaW4+6s3xXen4nVJWQctjZ1UF8Zbu9oDeFeU1aEO+xu76aiJEXGndbO0INuPkKpv/4hlUoahtGdyVCSSmAYnT3hLPry4hRm0NaVDocqAy1RrdVlRRjhdYwrD7d7w6yuohgDdrSGLS4z2BZ1Iz+xKtzf0hTWRwg72NTYwdTaMgxY3xgFX105Rgi+meMrMIM1O3pD8DiSCSMVNSumEgmSyXA/YdH4ZLjdG4XZG217xu49vnd5cW/hjYRQGEzz0XeAR9z9luj+/cDV7r54oOUqFIaQO2xdBs/+Ap79Oex4MYyf+RqYdwbMPTP01KrmijEjk3E6etJ09WTo6slw5Q+W8GJ0pNhh9RVk3Fm1vZUZdeW4w9pdbQBMqSkj486m3R1MqCrBga3NneDO+MoSPCukxpUX4x6a5arLinB3mjvC4coVxUkcaOvsobQ4iXs4Yg2gOJUAh850hqJkuN2dzgCQSBju3hc8o+uYyT2yAzSZCDGSjnaAFafCoeS9r7m3h+OO7jRlxUkMY3JNKb/7+GmH9twjIBRShKahNwAbCDua3+Huz2bNczZh5/N7zKweeBxY4O47BlquQiFPercglv0CHv4WdLeG8cnicBTT3DNh9mtDNxsiI0Bv8978SdU48Pzm0NR3+IQqHOfFLS3MnVCJZ22Nza6vwB1W7WhlVl05DqzZET7r02rDxarW7Wpj6rgy3An7nwhbH+6wuakDCFtu7iEYJ1SVALAl2lc0Mev+hKoScNjaHB43vjJM297SSV1FMe6wqy1s8WRv8UG4Too7NHf2UFmSwt2ZVFPKbz82SkMhKuJc4OuE/QU3ufsXzewaYIm732Vhe+q/gLOBNPBFd799f8tUKAyTpk2w4r7QW2t7YziKCaC4Al51RdgfMfPkcK1pERnxRkQo5INCIQbpbli/GFb9CR6+Hjqb6NuAn3QczHoNzDwFZpwCFeNjLVVEclMoSP50t8O6x2DNQ/DIt6GrOfTDBNBwJMw8NYTEzFOhenK8tYoIoFCQ4dTTCRsfhzV/CUHx0gN7mptSpXDsxSEgZpwMtbO041okBgoFiU+6BzY/FQJizUPw4m9Cp30AyaJwDepTPwpTXwWTXzGoy42KyMujUJCRI5OBbc+FgFi/OBzh1NO5Z3pROcx/C0xeAFMWwKRjoaQqvnpFxiCFgoxsLdtCk9OGpeHiQV2t4cpyvVKlMPeMEBATjwnnS6jpSeSQKRRk9GneDJueDE1Pm5+BF36z54xrgOKqEA6TjglBMelYmDA/HCYrIvs12FDQRXdl5KiaFIbDz9ozrqsVtj4Hm5+GLc/AU3eEJqjeHdkA4+dGIXEMTDw2BEfNNG1ViBwCbSnI6OMOjWvC1sSWZ0JgrLgPejr2zJNIhavRTTwa6ueFYfw8qJ6isJCCpC0FGbvMwv6F2lkw/817xnc0hb6cercqnv4ZrH1ozzkUELoPLyqDw8+B+sOh4fDwt24OFJUO9ysRGXEUCjJ2lFbDjJPCAPCWb4StiuZNsP3F0OHf9hXh73O/2tOFeK9USegMcPzcaJgT/tZMg0Ry+F+PSAwUCjK2mYUmo+opcFi/jsS62mDHCtj+QhQaK+CFe+Gl+/ddTlE5zHl9CIn6eXuCo3y8mqNkTFEoSOEqLofJx4Uhmzu0bAkh0Te8BCvuh+fv3nveRDKcX9F/62L8XB0VJaOSQkGkP7M9R0LNes3e09I9sHtt1AyVNTz7i32bo5LFMP3ErMCIhnEzIFU8fK9H5CAoFEQORjIFdYeFgTfuPa2rDXaujILiRXj0Btj4BKx9eE83H30s7AM56gKomw21s/f8La0erlcjsg8dkioyHNp27tmq2LkqXCu7pyOcnJcrMIor4YizQ0jUzgpbF+NmQPXUEEwiB0lnNIuMFh1NsGtVCItdq0J35D0d0N2xb5MUQLIkHCmVKoWFV+wJjL7QKBr+1yAjnkJBZCzo6YKm9dC4du9h+f9BZzM5r1bcGxpHvmnvwFBoFDSdvCYyFqSKs/Zh5NDTBU0bcoTGPfDk7ew3NI44Z+/AqJkezslIleT1JcnIplAQGc1SxWEHdd3s3NOzQ2P3Oti1Jvx9/te5j5iCcNTUlONh3PS9A2PczBAaOvN7TFMoiIxlBwqNdDc0bdwTGtlbG8/dtfd1L3oli8PFkaqnRsOUcNnV3ttVk9VENYopFEQKWbIIameGIZd0T+gmpH9gNK7dt2vzbJUTo7DICoq+AImGorL8vS45ZAoFERlYMhU1I02HmafsO90dOnaH4GjaELY6mjbuub1zJbzwf5BJ7/vYRAoa5u8dFH1DFCC6At+wUyiIyKEzg7JxYZgwf+D5ulqhKTs4sgJk9YPhxL9cO8UtGfqa6g2Lqn6hUT0FymrV/9QQUiiISP4VV0D93DAMpKcz2uLot7XR+3f1n/e+ZGsvS4ST/PrCon9T1VQor4dEIn+vbwxRKIjIyJAq2XOdjIGke0JnhblCY8V9ITiAfbc6LDSBVU+N9m9kb21Efysn6mxxFAoiMpokU1AzNQy8Ovc8mQy0bc+9j6NpYziHI/sqfdn6mqcm7x0aVZNDaFQ2QEn1mG6uUiiIyNiSSEDlhDBMeWXuedyhfVfu0GjaANtegOfv2fta4H0snLdR2RCCoiL6Wzlh79uVE0IfVqMsQBQKIlJ4zKC8LgyTjhl4vo6mPUdWtWyD1q2h+aplW/jbuBbWL4bW7eTcUZ4qiwJiIlRNhMpJWX8nReMnjah9HnkNBTM7G/gGkARudPdr+02/HPhPYEM06jp3vzGfNYmIDFppdRgajtj/fOkeaNsRhUY09N3eEoZtL8CqB8MhvPuwPTvJq3qHidFWR++WxySoqM/7pWHzFgpmlgSuB84E1gOLzewud1/Wb9b/dfer8lWHiEjeJVPhS7xq4oHn7W4PIdG8BVo2h7/Nm/YcebXt+dANSa6mq9rD4COPD339WfK5pXACsMLdVwKY2e3A+UD/UBARKRxFZQc+ygr2hEf21saMHCcQDrF8hsJUYF3W/fXAiTnmu8jMXge8AHzM3df1n8HMrgSuBJgxY0YeShURGWEGGx5DLJ97NnLtcu+/J+ZXwCx3Pw64D/ifXAty9xvcfaG7L2xoaBjiMkVEpFc+Q2E9MD3r/jRgY/YM7r7D3Xu7Yfwe8Ko81iMiIgeQz1BYDMwzs9lmVgxcAtyVPYOZTc66ex7wXB7rERGRA8jbPgV37zGzq4DfEA5JvcndnzWza4Al7n4X8GEzOw/oAXYCl+erHhEROTBdo1lEpAAM9hrNI+MUOhERGREUCiIi0kehICIifUbdPgUz2wasOcSH1wPbh7CcsULrZV9aJ7lpvexrtKyTme5+wBO9Rl0ovBxmtmQwO1oKjdbLvrROctN62ddYWydqPhIRkT4KBRER6VNooXBD3AWMUFov+9I6yU3rZV9jap0U1D4FERHZv0LbUhARkf1QKIiISJ+CCQUzO9vMlpvZCjO7Ou564mJmq83saTN7wsyWROPqzOx3ZvZi9Lc27jrzzcxuMrOtZvZM1ric68GCb0afnafM7Pj4Ks+fAdbJ581sQ/R5ecLMzs2a9ulonSw3s7PiqTr/zGy6mT1gZs+Z2bNm9pFo/Jj8vBREKGRdL/oc4CjgUjM7Kt6qYvU37r4g69jqq4H73X0ecH90f6y7BTi737iB1sM5wLxouBL49jDVONxuYd91AvC16POywN3vAYj+fy4Bjo4e863o/2ws6gE+4e7zgZOAf4he/5j8vBREKJB1vWh37wJ6rxctwfnsuerd/wAXxFjLsHD3BwndtWcbaD2cD/zAg0eAcf2uBTImDLBOBnI+cLu7d7r7KmAF4f9szHH3Te7+1+h2M+G6L1MZo5+XQgmFXNeLnhpTLXFz4LdmtjS69jXARHffBOEfAJgQW3XxGmg9FPrn56qoGeSmrKbFglwnZjYLeCXwKGP081IooTCY60UXilPd/XjCJu4/mNnr4i5oFCjkz8+3gTnAAmAT8F/R+IJbJ2ZWCfwM+Ki7N+1v1hzjRs26KZRQOOD1oguFu2+M/m4Ffk7Y5N/Su3kb/d0aX4WxGmg9FOznx923uHva3TOE66j3NhEV1DoxsyJCIPzI3e+MRo/Jz0uhhMIBrxddCMyswsyqem8DbwSeIayL90SzvQf4ZTwVxm6g9XAX8O7oqJKTgN29zQZjXb+28AsJnxcI6+QSMysxs9mEnaqPDXd9w8HMDPg+8Jy7fzVr0pj8vOTtGs0jyUDXi465rDhMBH4ePuOkgB+7+71mthi4w8z+FlgLvC3GGoeFmd0GnA7Um9l64HPAteReD/cA5xJ2prYBVwx7wcNggHVyupktIDR/rAb+DiC63vodwDLC0Tn/4O7pOOoeBqcC7wKeNrMnonH/zBj9vKibCxER6VMozUciIjIICgUREemjUBARkT4KBRER6aNQEBGRPgoFERHpo1AQGQQzW9Cv2+jzhqoLdjP7qJmVD8WyRF4unacgMghmdjmw0N2vysOyV0fL3n4Qj0mO4ZPFJEbaUpAxxcxmRRdD+V50QZTfmlnZAPPOMbN7ox5j/2RmR0bj32Zmz5jZk2b2YNQ1yjXA26MLzbzdzC43s+ui+W8xs29HF2JZaWanRT2KPmdmt2Q937fNbElU1xeicR8GpgAPmNkD0bhLLVwI6Rkz+3LW41vM7BozexQ42cyuNbNlUQ+mX8nPGpWC4+4aNIyZAZhF6HZhQXT/DuCyAea9H5gX3T4R+H10+2lganR7XPT3cuC6rMf23SdcnOZ2Qu+Y5wNNwLGEH11Ls2qpi/4mgT8Ax0X3VwP10e0phC4TGghdkfweuCCa5sCi3mUBy9mztT8u7nWvYWwM2lKQsWiVu/f2UbOUEBR7ibpBPgX4SdSfzXeB3s7f/gLcYmbvJ3yBD8av3N0JgbLF3Z/20LPos1nPv8jM/go8TrhiWa6r/70a+IO7b3P3HuBHQG/35mlCT50QgqcDuNHM3kroY0fkZSuIDvGk4HRm3U4DuZqPEkCjuy/oP8HdP2BmJwJvAp6IOoQb7HNm+j1/BkhFPYl+Eni1u++KmpVKcywnV1/8vTo82o/goZPHE4A3EHr9vQp4/SDqFNkvbSlIQfJwkZRVZvY26LvY+iui23Pc/VF3/yywndA3fjNQ9TKeshpoBXab2UTCRY56ZS/7UeA0M6uPrnl8KfDH/guLtnRqPFwz+aOEi+CIvGzaUpBC9k7g22b2GaCIsF/gSeA/zWwe4Vf7/dG4tcDVUVPTlw72idz9STN7nNCctJLQRNXrBuD/zGyTu/+NmX0aeCB6/nvcPdf1LaqAX5pZaTTfxw62JpFcdEiqiIj0UfORiIj0UfORjHlmdj3h6lnZvuHuN8dRj8hIpuYjERHpo+YjERHpo1AQEZE+CgUREemjUBARkT7/H4nRIqP9YtMNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb973908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators_1.png' )\n",
    "\n",
    "pyplot.show()\n",
    "#n_estimators在219时停止"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.对树的最大深度（可选）和min_children_weight进行调优（可选）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "（1）粗调，步长为2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60231, std: 0.00320, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60268, std: 0.00324, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60254, std: 0.00324, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58876, std: 0.00418, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58967, std: 0.00375, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58949, std: 0.00349, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58920, std: 0.00325, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.58937, std: 0.00384, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58880, std: 0.00361, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.60216, std: 0.00386, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.59912, std: 0.00576, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.59562, std: 0.00382, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.5887620837531603)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_\n",
    "#best {'max_depth': 5, 'min_child_weight': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([222.78160009, 203.41740003, 228.76479993, 316.97840004,\n",
       "        284.66064005, 289.50380001, 369.91674004, 374.26758003,\n",
       "        349.58791995, 419.4117599 , 412.62956004, 406.40495992]),\n",
       " 'mean_score_time': array([0.57039995, 0.4947999 , 0.61420007, 0.8737999 , 0.82519994,\n",
       "        0.69400001, 1.74179997, 1.0954    , 1.06079998, 2.21120019,\n",
       "        2.16280003, 1.84620013]),\n",
       " 'mean_test_score': array([-0.60231111, -0.60267855, -0.60254479, -0.58876208, -0.58966823,\n",
       "        -0.58948975, -0.58919785, -0.58936631, -0.58879754, -0.60216167,\n",
       "        -0.59912006, -0.5956157 ]),\n",
       " 'mean_train_score': array([-0.57949952, -0.58053343, -0.58076776, -0.51338803, -0.51893314,\n",
       "        -0.52194029, -0.41203775, -0.43241468, -0.44481989, -0.29008614,\n",
       "        -0.33474867, -0.36146092]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'rank_test_score': array([10, 12, 11,  1,  6,  5,  3,  4,  2,  9,  8,  7]),\n",
       " 'split0_test_score': array([-0.59729775, -0.59770222, -0.5976163 , -0.58164061, -0.58311838,\n",
       "        -0.58350541, -0.5835627 , -0.58326285, -0.58301462, -0.59597775,\n",
       "        -0.5913763 , -0.59045568]),\n",
       " 'split0_train_score': array([-0.5807981 , -0.58204125, -0.58219045, -0.5146005 , -0.52134364,\n",
       "        -0.52458694, -0.41473745, -0.43451401, -0.4476738 , -0.29145456,\n",
       "        -0.33524207, -0.36160542]),\n",
       " 'split1_test_score': array([-0.60106583, -0.60076115, -0.60096747, -0.58687398, -0.58802195,\n",
       "        -0.58853755, -0.58807372, -0.58898465, -0.58712017, -0.60076562,\n",
       "        -0.59487891, -0.59420218]),\n",
       " 'split1_train_score': array([-0.579754  , -0.58032957, -0.58110724, -0.51360895, -0.51958685,\n",
       "        -0.52179084, -0.4120803 , -0.43257833, -0.44358349, -0.28827489,\n",
       "        -0.33417783, -0.36070062]),\n",
       " 'split2_test_score': array([-0.60237856, -0.6030287 , -0.60218843, -0.58986775, -0.59163298,\n",
       "        -0.58951364, -0.59118543, -0.59189609, -0.58981765, -0.60502669,\n",
       "        -0.60087598, -0.596142  ]),\n",
       " 'split2_train_score': array([-0.58013167, -0.58090438, -0.5803785 , -0.51342309, -0.51849693,\n",
       "        -0.52130926, -0.41118857, -0.43112813, -0.44368944, -0.28967423,\n",
       "        -0.3342522 , -0.3602344 ]),\n",
       " 'split3_test_score': array([-0.60704061, -0.60708537, -0.60696721, -0.5922439 , -0.59198498,\n",
       "        -0.59250983, -0.59011711, -0.58800923, -0.59015468, -0.60179936,\n",
       "        -0.6001957 , -0.59505273]),\n",
       " 'split3_train_score': array([-0.57909986, -0.58028792, -0.58072622, -0.51396996, -0.51867226,\n",
       "        -0.52189379, -0.41094505, -0.43140028, -0.44348998, -0.28838499,\n",
       "        -0.33300857, -0.36129872]),\n",
       " 'split4_test_score': array([-0.60377322, -0.60481598, -0.60498528, -0.59318552, -0.59358404,\n",
       "        -0.59338349, -0.59305149, -0.59468037, -0.59388213, -0.60724045,\n",
       "        -0.60827618, -0.60222793]),\n",
       " 'split4_train_score': array([-0.57771396, -0.57910405, -0.57943639, -0.51133766, -0.51656602,\n",
       "        -0.52012061, -0.41123737, -0.43245264, -0.44566274, -0.292642  ,\n",
       "        -0.33706269, -0.36346542]),\n",
       " 'std_fit_time': array([ 8.89533078,  5.54670496, 27.483118  , 13.87134902,  6.73910944,\n",
       "        14.879706  , 17.31416869, 13.53533857, 19.48681483,  6.98041895,\n",
       "         7.94915297, 12.38646782]),\n",
       " 'std_score_time': array([0.13526509, 0.10934973, 0.09624009, 0.1297867 , 0.13316079,\n",
       "        0.01698235, 0.92421325, 0.05187913, 0.02305128, 0.06366913,\n",
       "        0.37235438, 0.22405217]),\n",
       " 'std_test_score': array([0.00319992, 0.00324202, 0.0032379 , 0.00417593, 0.00374624,\n",
       "        0.00349287, 0.0032454 , 0.00384381, 0.0036052 , 0.00385539,\n",
       "        0.00576216, 0.00381918]),\n",
       " 'std_train_score': array([0.00104868, 0.00095458, 0.00090167, 0.00110116, 0.0015556 ,\n",
       "        0.00146552, 0.00140327, 0.00119328, 0.00163893, 0.00171751,\n",
       "        0.00135635, 0.00110884])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588762 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPW9//HXZ5YkZAOEgCAiiiggUJQIKATBSoLaS2ttcbl6i631ttZae38ute3Vaq3L7W21i2211uW2blVLiy1KcIMAIptBEaygokYsYFiSTEKSmfn8/jgnw2SYJJNMJpPl83w85sHMOd9z5jMnzHnPWb+iqhhjjDEd5Ul3AcYYY3o2CxJjjDFJsSAxxhiTFAsSY4wxSbEgMcYYkxQLEmOMMUmxIDHGGJMUCxJjjDFJsSAxxhiTFF+6C+gKgwcP1lGjRqW7DGOM6VE2bNjwqaoWtNWuTwTJqFGjWL9+fbrLMMaYHkVEPkikne3aMsYYkxQLEmOMMUmxIDHGGJMUCxJjjDFJsSAxxhiTFAsSY4wxSbEgMcYYk5Q+cR2JMV0lGApzMBjmYGOIevdf5xGmPhiivtEdFnSGBUNhPB7BK4LXc+jhiX7tPj+8Hc3buePbnF+kHZF2IpLuRWd6MAsS0yupKsGwRlbizor90Ar90LBwszYHm1b2USv92EBorU0wrOn+6B3iEZoHjgheb2vBdKi9L6adxyP4YgLMI1HDPIJXSLBdvPmRYLuW5gc+j6dZEMf7HHGDOGp6C+JDLEhMyqkqDaHw4b/KY1bqB+Ot6GNX9sFQ3ABoaHodtdJPZp2e4fOQ5fOQ5feS6feQ5fOS5feS5feQm+ljUI6HTL/XHe4h0/23qU2W30tm0/S+w4c1tfN5PIRVCYWjHqqE3X+jhzvtOLxdTNsW5xdWQgrhsBOy4ZhpguHm8zu8HQm2UxpDYeoaoz9H0/uGCSvx39edPnZ+PSGbRcB3WOA4IRYdTJEQigluX0wget1giw46byTADgVx/HbN57dwxigG52am9PNbkPQxqkp9MNz8F3UwFPPLPeoXeDAcszKPFwBOm8i/sW2CITSJlcGhla/7r+/Qyj0vy0dBXmbzNu5KP3pYZuxKvIWQyPJ7yfA6X1bTPag6YRIMhwmHaTFwWg4md9qoIE40EA9v174gbv5jgIR/NATDYeqD0T8SaOXHRfMfE7HtvnDyURYkvVk43PRLPc6ulahf100r8fp4K/s4u2naComOEqHZSjzyK9tdSffv5ycrL/PQyrvZr3SnTWbUSjx2Zd/stfs+mT5Pn99t0NeJOL/AvR5vuksxLbAgacWHlbVUBuqb7Uapb2HXSuwv8MN2ycQJiYYkVuoeodmv68yYlfLA7Ixmv+CbfpU3tT1sZd/0K/6wkDjUJsNrK3VjzOEsSFpx0+LNvPLPPW2283mk2W6TzJgV9OBc32H7x2PbxP8Vf+h1Zkwbv9cO8BljuoeUBomIzAN+AXiBB1T1zjhtFgA/AhTYpKoXu8O/AvzQbXabqj4iItnAU8BoIAQ8q6rfS1X93558BP9ROJysflnur/g4B0x9HnxeuxzHGNN3pSxIRMQL3AvMBSqAdSKyWFW3RLUZA9wIzFDVfSIyxB1+BHAzUIgTMBtEZDFQD/yvqr4sIhnAiyJytqo+l4rPMOSBe8h99VWyTzuN3KKZ5BYV4S84KhVvZYwxPVYqt0imAttV9T0AEXkC+DywJarN14F7VXUfgKrudoeXAMtUda877TJgnqo+Drzstm0QkY3AiFR9gIEXXoBvSAGBFWXUvPgiABmjR5M7cyY5s4rIPvVUPBkZqXp7Y4zpEVIZJEcBH0W9rgCmxbQ5AUBEVuHs/vqRqj7fwrTNNgVEZADwbzi7zlIit6iI3KIi5zqI99+nZsUKAmUr2ff44+x95BGkXz9ypk4lZ1YRubNmkXH00akqxRhjuq1UBkm8I8GxVxP4gDHAbJwtizIRmdDWtCLiAx4Hftm0xXPYm4tcAVwBMHLkyPbWHjsvMo87jszjjmPQwoWEa2upXbeOmhVl1JSVUbN8ObuAjGOOIWfWLHKbtlayspJ6X2OM6QlSGSQVQPRP9BHAzjht1qhqI/C+iPwTJ1gqcMIletpXol7fD2xT1XtaenNVvd9tR2FhYadeG+vJzib3jDPIPeMMABp27KCmbCU1ZSvY/+c/s++Pf0QyM8meOpXcoiJyimaSMWqUnWVljOmVRJO55Li1GTtbDe8AnwU+BtYBF6vqW1Ft5gEXqepXRGQw8DowGfcAO3CK23QjMEVV94rIbcA44MuqmtCFGIWFhbp+/fpO+mStCx88SO269dSUrSCwooyGHTsA8B99tBMqs4rImToVT3Z2l9RjjDEdJSIbVLWwzXapChK3iHOAe3COfzyoqj8RkVuB9aq6WJyf6D8D5uGczvsTVX3CnfarwPfdWf1EVR8SkRE4x07exjmDC+DXqvpAa3V0ZZDEavjoI2rKygisKCPw2mtoXR2SkUF2YSE5RUXkzioi47jjbGvFGNPtdIsg6S7SGSTRwg0N1K1fHzm20vDuuwD4hw+PhEr2tOl4c3PSXKkxxliQNNNdgiRW48cfu8dWyqh99VXCtbXg95N9yinkzioip6iIzDFjbGvFGJMWFiRRumuQRNOGBmo3vk5gZRk1K8qof+cdAHxHHklu0UxyiorIOf10vLm5aa7UGNNXWJBE6QlBEqvxX/8isHIlNSvKCKxeTbimBnw+sidPjpxinHniiba1YoxJGQuSKD0xSKJpYyN15eWR3WD1W7cC4CsoiBxbyTntNLz9+6e5UmNMb2JBEqWnB0msxt27Caxc5ZxivGo14aoq8Hrp95nPOKEys4is8eMQj91M0hjTcRYkUXpbkETTYJC6N96MXLdy8C3nMh3voEHOPcGKisiZcTq+gQPTXKkxpqexIInSm4MkVrCy8tCxlZUrCR04AB4P/SZOjOwGy5owwbZWjDFtsiCJ0peCJJqGQhzcvNm5bmVlGQffeBNU8Q4cSM6MGe5usJn4jjgi3aUaY7ohC5IofTVIYgX37SOwcpVzinHZSkJ794IIWSedFLlupd+kSYjX+sY2xliQNGNBcjgNhzn41pbIdSt1mzZBOIynf39yZ5xOTtEscmfOwFdQkO5SjTFpYkESxYKkbaH9+wmsXu2cYryyjNCeTwHIHD+O3KJZ5BbNpN/kyYgvpb0zG2O6EQuSKBYk7aPhMPVvvx25NX7d6+UQCuHJyyPn9NMjV9r7hw5Nd6nGmBSyIIliQZKcUFUVgVfXOKcYl60kuGsXAJknnhi5biX7lJMRvz/NlRpjOpMFSRQLks6jqtS/s41A2QpqVpRRu3EjBIN4cnLIPm26sxtsVhH+YcPSXaoxJkkWJFEsSFInVFND7Zo1kVvjBz/5BICM40dHQqXflCl4MjLSXKkxpr0sSKJYkHQNVaXh3XfdiyHLqF23Hm1sRLKzyZk2zT3FeBYZI45Kd6nGmARYkESxIEmPcCBA4LW1kVOMGysqAMg49thIqGSfWognMzPNlRpj4rEgiWJBkn6qSsP7O5xjK2UrqV27Fm1oQLKyyJ42NXKKccYxx6S7VGOMy4IkigVJ9xOuq6N27drIKcaNH3wIgP+YkZFQyZ46FU+/fmmu1Ji+y4IkigVJ99fwwQeRUKl9bS168CCSkUH21KnudSuzyDh2lHXkZUwX6hZBIiLzgF8AXuABVb0zTpsFwI8ABTap6sXu8K8AP3Sb3aaqj7jDpwAPA/2AJcB3tI0PYUHSs4Tr66ldtz5yinHD++8D4B8xgpyimeQWzSJn+jQ82dlprtSY3i3tQSIiXuAdYC5QAawDLlLVLVFtxgB/Bs5U1X0iMkRVd4vIEcB6oBAnYDYAU9w2a4HvAGtwguSXqvpca7VYkPRsDRUVBMqcA/aB115Da2sRv59+hVMipxhnjB5tWyvGdLLuECSnAT9S1RL39Y0AqnpHVJv/Ad5R1Qdipr0ImK2q/+m+vg94xX28rKpj47VriQVJ7xFuaKBuw4bIKcb127YD4Bs+jNyZTn8r2dOn483NTXOlxvR8iQZJKu/AdxTwUdTrCmBaTJsTAERkFc7urx+p6vMtTHuU+6iIM/wwInIFcAXAyJEjO/whTPfiycgg57TTyDntNLjhehp37qSmbCWBlWVU/eMf7P/zn8HnI/uUUyK3xs884QTbWjEmhVIZJPG+ubGbPz5gDDAbGAGUiciEVqZNZJ7OQNX7gfvB2SJJrGTT0/iHD2fgBQsYeMECtKGB2tfLI6cY7/7fn8H//gzf0KHOsZWZReScfhre/Px0l21Mr5LKIKkAjo56PQLYGafNGlVtBN4XkX/iBEsFTrhET/uKO3xEG/M0fZRkZJAzbSo506Yy5Nprady1yzm2UraS6qWlHHj6GfB66Xfy5MhusMxx42xrxZgkpfIYiQ/nYPtngY9xDrZfrKpvRbWZh3MA/isiMhh4HZjMoQPsp7hNN+IcbN8rIuuAbwOv4Rxs/5WqLmmtFjtGYrSxkbpNmyKnGNdv2QqAt2CwEypFM8k5/XS8AwakuVJjuo+0HyNR1aCIXAUsxTn+8aCqviUitwLrVXWxO65YRLYAIeA6Va10P8CPccIH4FZV3es+/yaHTv99zn0Y0yrx+8kuLCS7sJAh372G4J491KxcRaBsBdUvvcSBRYvA46HfpEnkzCoit2gWWSeNRzyedJduTLdnFySaPk+DQerefDNyivHBzZsB8B5xBDkzZzjXrcycgW/gwDRXakzXSvvpv92JBYlpj2BlJYFVq9xTjFcS2r8fRMiaOJHcImc3WNbEiYjXm+5SjUkpC5IoFiSmozQU4uBbb7n9razg4BtvgireAQPImTHDOcV4xgx8gwenu1RjOp0FSRQLEtNZgvv2EVi12jnFeOUqQpWVAGSddJJ7bKWIfpMmIb5UnhBpTNewIIliQWJSQcNhDm7ZGrlupa68HMJhPPn55Mw43blupWgm/iFD0l2qMR1iQRLFgsR0hdCBAwRWr3autC8rI7hnDwCZ48aRO3Om0+3w5MmI35/mSo1JjAVJFAsS09VUlfq333ZCZcUKal9/HUIhPLm5zi1e3N1g/iOPTHepxrTIgiSKBYlJt1B1NYFXX42cYhzctQuAzBNPJK94LvklJWQef3yaqzSmOQuSKBYkpjtRVeq3bSNQVkb1iy9R9/rroErGcceRV1LshMqJJ9qtW0zaWZBEsSAx3Vnjrt1Uv7CM6tJl1K5bB+Ew/mNGkl9cTF5xCVkTTrJQMWlhQRLFgsT0FMHKSqpfeJHq0lICa9ZAKIR/+HDyiovJKymm32c+Y7dtMV3GgiSKBYnpiYL79lHz0stUlS4lsPpVaGzEN3QoecXF5JcU0+/kk+3qepNSFiRRLEhMTxeqqqLmlVeoWlpKoKwMbWjAO3gweXPPIr+khOzCQrsI0nQ6C5IoFiSmNwnVBAisWE7V0lJqVqxA6+rwDhxI3lmfJa+4hJzp0+xaFdMpLEiiWJCY3ipcV0dNWRnVS0upefllwrW1ePLzyTvzTPJKismZMQNPRka6yzQ9lAVJFAsS0xeE6+sJrFpF9dJSql96iXB1NZ7cXHLnzCGveC65RUV4srLSXabpQSxIoliQmL5GGxoIrFlD1dKl1LzwIqEDB5DsbHLPmEV+SYkTKjk56S7TdHMWJFEsSExfpo2N1K5bR9XSUqqXLSO0dy+SmUnurCLyikvInTMbb25uuss03ZAFSRQLEmMcGgpRu2GDs/urtJTgnj2I30/OjBnklZSQd+YcvP37p7tM001YkESxIDHmcBoOU1deTvXSUqpKSwl+8gn4fORMn05eSTF5Z51l3Qv3cd0iSERkHvALwAs8oKp3xoxfCPwU+Ngd9GtVfcAddxdwrjv8x6r6pDv8s+40HqAGWKiq21urw4LEmNapKgfffJPq0lKqlpbS+NFH4PWSPfVU51YtZ52Fr6Ag3WWaLpb2IBERL/AOMBeoANYBF6nqlqg2C4FCVb0qZtpzgWuAs4FMYDlwpqpWicg7wOdVdauIXAlMVdWFrdViQWJM4lSV+q1bnWMqS5fSsGMHiJA9ZYpzq5biuXb7+z4i0SBJ5U17pgLbVfU9VW0AngA+n+C044HlqhpU1QCwCZjnjlMg333eH9jZiTUb0+eJCFnjxzPku9dw3HNLOHbx3xh85ZWEDuxn1+23s332HHZceBGVDz1M48cftz1D0+ul8p4KRwEfRb2uAKbFaXe+iMzC2Xr5rqp+hBMcN4vIz4FsYA7QtCVzObBEROqAKmB6R4prbGykoqKCgwcPdmRy08NkZWUxYsQI/HbFd7uICFknnEDWCSdQ8O2rqH/vvcjur9133cXuu+4ia8IE5/b3xcVkHHNMuks2aZDKXVtfBkpU9XL39aU4u6G+HdVmEFCjqvUi8g1ggaqe6Y77AfBlYA+wG1irqr8Qkb8Ad6nqayJyHXBi03vEvP8VwBUAI0eOnPLBBx80G//++++Tl5fHoEGD7BbdvZyqUllZSXV1Nccee2y6y+k1Gj78MBIqB998E4DMsWPJLykmr6SEzOOOS3OFJlnd4RjJacCPVLXEfX0jgKre0UJ7L7BXVQ8791BEHgP+hHOcZY2qjnaHjwSeV9XxrdUS7xjJ1q1bGTt2rIVIH6GqvP3224wbNy7dpfRKjR9/TNWyZVQvLXU66gIyxxxP3lw3VE4YY9+1HijRIEnlrq11wBgRORbnrKwLgYujG4jIMFX9xH05H9jqDvcCA1S1UkQmAZOAUrddfxE5QVWbDuRv7WiB9h+777C/dWr5jzqKQQsXMmjhQhp37aK6dBnVpaV8+tvf8ulvfkPGqFHOdSrFc8kaP97+Hr1MyoJEVYMichWwFOf03wdV9S0RuRVYr6qLgatFZD4QBPYCC93J/UCZ+5+tCrhEVYMAIvJ14BkRCQP7gK+m6jMYY9rPP3QoR1x6CUdcegnBPXuofvFFqpYupfKBB6i87z78I0ZEuhTOmjjRQqU3UNVe/5gyZYrG2rJly2HDutK+ffv03nvv7dC0d999twYCgU6uqHOcccYZum7dug5Nu2jRIn3rrbfaNa+6ujo99dRTddKkSTp+/Hi96aabWmyb7r95X9e4d6/ue+op/eDyr+uWkybolhPH6juz5+i/br9dAxs2aDgUSneJJgbOj/4217HWZ2ea7N+/n9/85jcdmvaee+6htra2kytKv7/+9a9s2bKl7YZRMjMzeemll9i0aRPl5eU8//zzrFmzJkUVmmT4Bg5kwJe+xMjf388Jq1Yy7M47yBo7ln2PPc4HF/8728+Yzb9u/TGB19aioVC6yzXtYF2qAbc8+xZbdlZ16jzHD8/n5n87qcXx3/ve93j33XeZPHkyc+fOZciQIfz5z3+mvr6e8847j1tuuYVAIMCCBQuoqKggFArx3//93+zatYudO3cyZ84cBg8ezMsvvxx3/rm5uXzrW9/ihRdeYODAgdx+++1cf/31fPjhh9xzzz3Mnz+fHTt2cOmllxIIBAD49a9/zemnn86iRYu49957WbZsGf/6178444wzWLFiBUfGuQitrq6Oyy67jC1btjBu3Djq6uoi40pLS7n55pupr69n9OjRPPTQQ+Tm5jJq1CguuOCCSO2PPfYYu3fvZvHixSxfvpzbbruNZ555BoCnnnqKK6+8kv379/OHP/yBoqKiZu8vIuS6NxxsbGyksbHRdpX0AN7+/RnwhS8w4AtfIFRTQ80ry6leupT9zzzDvscewztoEHlnnUVe8Vxypk61jrq6OdsiSZM777yT0aNHU15ezty5c9m2bRtr166lvLycDRs2sGLFCp5//nmGDx/Opk2b2Lx5M/PmzePqq69m+PDhvPzyyy2GCEAgEGD27Nls2LCBvLw8fvjDH7Js2TIWLVrETTfdBMCQIUNYtmwZGzdu5Mknn+Tqq68G4LzzzuPII4/k3nvv5etf/zq33HJL3BAB+O1vf0t2djZvvPEGP/jBD9iwYQMAn376KbfddhsvvPACGzdupLCwkJ///OeR6fLz81m7di1XXXUV11xzDaeffjrz58/npz/9KeXl5YwePRqAYDDI2rVrueeee7jlllsA2LlzJ+ecc05kXqFQiMmTJzNkyBDmzp3LtGnxLlcy3ZU3N5f+nzuXEb/6JSesXsVR99xNzrSpHHj2WT762uVsm1nEzh/8gJrly9GGhnSXa+KwLRJodcuhK5SWllJaWsrJJ58MQE1NDdu2baOoqIhrr72WG264gc997nOH/RpvTUZGBvPmOTcDmDhxIpmZmfj9fiZOnMiOHTsA5xf8VVddRXl5OV6vl3feeScy/a9+9SsmTJjA9OnTueiii1p8nxUrVkQCaNKkSUyaNAmANWvWsGXLFmbMmAFAQ0MDp512WmS6pnledNFFfPe7321x/l/84hcBmDJlSqTu4cOHs2TJkkgbr9dLeXk5+/fv57zzzmPz5s1MmDAhoeVkuhdPTg758+aRP28e4YMHCaxcSVVpKdVLSznwzF/w5OWRd+Yc8kpKnN4fMzPTXbLBgqRbUFVuvPFG/vM///OwcRs2bGDJkiXceOONFBcXR7Ym2uL3+yO7eDweD5nuF87j8RAMBgG4++67GTp0KJs2bSIcDpMV1Xvexx9/jMfjYdeuXYTDYTyeljde4+1KUlXmzp3L448/3uY0re2Kaqrb6/VG6m7JgAEDmD17Ns8//7wFSS/gycpydm+ddRbhhgYCq1dHen888LfFeLKzyZ09m7ySEnJnFeHp1y/dJfdZtmsrTfLy8qiurgagpKSEBx98kJqaGsBZie/evZudO3eSnZ3NJZdcwrXXXsvGjRsPmzYZBw4cYNiwYXg8Hv74xz8Scg9wBoNBLrvsMh577DHGjRvXbJdUrFmzZvHoo48CsHnzZt544w0Apk+fzqpVq9i+3bkxc21tbbMtnieffDLyb9OWSkc+1549e9i/fz/gHK954YUXGDt2bLvmYbo/T0YGebNnM/yO2zlhZRlHP/AA+eeeS+DVV/n4O9/hndNnUHH1dzjwj38Qqgmku9w+x7ZI0mTQoEHMmDGDCRMmcPbZZ3PxxRdHVqi5ubn86U9/Yvv27Vx33XV4PB78fj+//e1vAbjiiis4++yzGTZsWKvHSdpy5ZVXcv755/PUU08xZ84cctyuV2+//XaKioooKipi8uTJnHrqqZx77rlxrwr/5je/yWWXXcakSZOYPHkyU6dOBaCgoICHH36Yiy66iPr6egBuu+02TjjhBADq6+uZNm0a4XA4stVy4YUX8vWvf51f/vKXPP300y3WvXPnTi6//HKWLFnCJ598wle+8hVCoRDhcJgFCxbwuc99rsPLxHR/4veTO3MGuTNncOTNN1G7fgPVpUudK+tLS5GMDHJmziS/pJjcOXPw5ue3PVOTlDZvkSIio4EKde6HNRvnKvP/U9X9XVBfp2jpFil2u4z0GDVqFOvXr2fw4MFd+r72N+/dNBSirrycqqVLqV5aSnDXLvD7yTn9NPKLS8g9c4511NVOnXmLlGeAQhE5HvgDsBh4DDin1amMMaYLiddL9pQpZE+ZwtDvfY+Db7zh9KlSWsony38AXi8506Y5t2o567P4Bg1Kd8m9RiJbJBtV9RT3TrsHVfVXIvK6qp7cNSUmrzdvkUybNi2y66jJH//4RyZOnNip77N06VJuuOGGZsOOPfZYFi1a1Knvk0q95W9u2kdVOfjWFqqXLqWqdCmNH3wIHg/ZhYVul8Jz8Q8dku4yu6VOu/uviLwG3AP8APg3VX1fRDarao85LaY3B4lJnP3NjapS/847TqgsLaXh3XdBhH4nn+zc/r64GP+wYekus9vozF1blwHfAH7ihsixOLd0N8aYHkVEyDrxRLJOPJGCq6+mfvv2yHUqu+64k1133EnWZyY5/dQXF5Nx9NHpLrlHaFd/JCIyEDhaVd9IXUmdz7ZIDNjf3LSuYccOqkqXUb10KQffeguArPHjI7e/z+yDnaJ1Wp/tIvKKiOSLyBE4XeA+5HaBa4wxvUbGqFEMvuLrHPvM04x+YRlDrr8e8fvZc/fdvHf2Obw3//Psufde6t1ro8whieza6q+qVSJyOfCQqt4sIj1qi8QYY9ojY8QIBn31MgZ99TIaP/mE6mXLqFpayqe/vpdPf/VrMo47LtKnSuaJJ/b5G4UmcmW7T0SGAQuAv6e4nj6jt95Gfvbs2cTuRkxU7G3kE53XqFGjmDhxIpMnT6awsM2tcGPaxT9sGEf8x38w6tE/cfzyVxh603/jKyig8r77ef8L5/FuyTx2/+xn1L25mfYcKuhNEgmSW3F6OXxXVdeJyHHAttSW1fv11iBJRkf6I2ny8ssvU15e3uEQMyYR/iFDOOLiiznmkYcZs7KMI2+9hYyjj6bywYfY8eUv8+5nz2LXnXdR+/rraDic7nK7TJu7tlT1KeCpqNfvAeensqgu99z34F9vdu48j5wIZ9/Z4mjrj6Rz+iMxJl18RxzBwAULGLhgAaH9+6l+6WWqly5l76OPsvfhh/ENHUpecTH5xXPpd8opiNeb7pJTJpGD7SNEZJGI7BaRXSLyjIiM6IriejPrj6Tz+iMREYqLi5kyZQr3339/En8VYzrGO2AAA754Hkff9ztOWL2K4f9zF1kTJ7D/ySf54NL/YNsZs/nkllsIrFmDtnEX654okYPtD+HcEuXL7utL3GFz25pQROYBvwC8wAOqemfM+IXAT4GP3UG/VtUH3HF3Aee6w3+sqk+6wwW4za0nBPxWVX+ZwOdoWStbDl3B+iNJrj+SVatWMXz4cHbv3s3cuXMZO3Yss2bNSmg5GdPZvHl59J8/n/7z5xOqCRBYsZyq0mUc+Ovf2P/4E3gHDiTvrM+SV1xCzvRpvaL3x0SCpEBVH4p6/bCIXNPWRCLiBe7FCZwKYJ2ILFbV2J3gT6rqVTHTngucAkwGMoHlIvKcqlYBC4GjgbGqGhaRHn9vA+uPJLn+SIYPHw44W1jnnXcea9eutSAx3YI3N4f8c84h/5xzCNfVUVNWRvXSUqqWPMf+p57Gk59P3plnkldS7HTUlZGR7pI7JJGD7Z+KyCUi4nUflwCVCUw3FdiCibk1AAAa8klEQVSuqu+pagPwBPD5BOsaDyxX1aCqBnCuX5nnjvsmcKuqhgFUdXeC8+xWrD+SzumPJBAIRKYJBAKUlpZap1amW/L060d+cTFH/ex/GbN6FSN+8xvy5syh+sUXqfjmlWw7fQYfX3c9VcuWET54MN3ltksiWyRfBX4N3A0osBrntiltOQr4KOp1BRCvM+3zRWQW8A7wXVX9CCc4bnYvfMwG5gBNWzKjgQtE5DxgD3C1qva4s8isP5LO6Y9k165dnHfeeYATgBdffHFkl54x3ZUnM9PpMvjMOWhDA4HXXqNq6VJqXniRqmefRbKzyT1jFvnFxeTOmoXH/W52V+26RUpkIpFrVPWeNtp8GShR1cvd15cCU1X121FtBgE1bl8n3wAWqOqZ7rgf4BwH2QPsBtaq6i9EpAa4WVV/JiJfxAmfww4eiMgVwBUAI0eOnPLBBx80G2+3y0gf64/EmPg0GKR23TqnT5VlLxCqrEQyM8mdVURecQm5c2bjzc3tsno67RYpLfivBNpU4BzLaDIC2BndQFUrVbXpHui/B6ZEjfuJqk5W1bmAcOjalQqcPlIAFuF0tHUYVb1fVQtVtbCgoCCBco0xJr3E5yPntNMY9qMfMWbFckb+3yMM+PKXqdv0Bjuvu45tp53OR9/4JvsX/ZXQgQPpLjeio13tJnI/gHXAGPduwR8DFwIXN5uJyDBV/cR9OR/Y6g73AgNUtVJEJuGERanb7q/AmcCDwBk4u8T6rJ7YH0nT2VfGmJaJ10vO1KnkTJ3K0O/fSF35Juf298tKqXnlFT7x+ciZPt3tU+WstPb+2NFdWx+q6sgE2p2D05eJF3hQVX8iIrcC61V1sYjcgRMgQWAv8E1VfVtEsoCN7myqgG+oark7zwHAo8BIoMYdt6m1Ouzuvwbsb256B1Xl4ObNkT5VGj/6CLxesk891elT5ayz8HXSXpikO7YSkWqcg+uHjQL6qWpHt2a6nAWJAfubm95HVal/++1IP/UN778PImRPmUJecTF5xXPxt3AxcSKSPkaiqnmqmh/nkdeTQsQYY3orESFr3DiGXHMNxy35B8c9u5jB3/oWoQMH2HX77WyfPYe6zW+lvA4LBGOM6QVEhMwxYygYM4aCq75F/XvvU/PyS2SNG5vy97YgMcaYXijzuGPJPO5rXfJeHT391ySpt95Gvqv7I/nnP//J5MmTI4/8/HzuuafVS5yMMZ3MgiRNemuQJKMj/ZGceOKJlJeXR+6anJ2dHbnS3RjTNdrctdXC2VsHgPXA/3P7J+nR7lp7F2/vfbtT5zn2iLHcMPWGFsdbfySd3x/Jiy++yOjRoznmmGPa/gMZYzpNIlskPweuw7l31gjgWpyr0J/AuSjQdID1R9J5/ZE0eeKJJ1q95b0xJjUSOdg+T1Wjb7Z4v4isUdVbReT7qSqsK7W25dAVrD+S5PojaZr/4sWLueOOO9pcNsaYzpVIkIRFZAHQdDvWL0WN65s93Xcy648kuf5IAJ577jlOOeUUhg4d2mIbY0xqJLJr69+BS3HuwLvbfX6JiPQDrmptQtMy64+kc/ojafL444/bbi1j0qTNLRL3YPq/tTB6ZeeW03dYfySd0x8JOCG1bNky7rvvvg4vC2NMx7V500YRGQH8CpiBsytrJfAdVa1IfXmdw+611b1YfyTG9Ayd2R/JQ8BiYDjOmVvPusOMMcaYhA62F6hqdHA8LCLXpKog0z7WH4kxJt0SCZJPReQSoOn0m4uAytSVZNrjtdde65L3KSkpoaSkpEveyxjTsySya+urwALgX8AnOKf/XpbKoowxxvQcbQaJqn6oqvNVtUBVh6jqF4AvdkFtxhhjeoCO3rTxvzq1CmOMMT1WR4Ok5UuRjTHG9CkdDRK7NUqSeutt5Lu6PxKAX/ziF0yYMIGTTjrJ+iIxJg1aDBIRqRaRqjiPapxrStokIvNE5J8isl1Evhdn/EIR2SMi5e7j8qhxd4nIZvdxQZxpfyUiNQl+zm6ntwZJMjrSH8nmzZv5/e9/z9q1a9m0aRN///vf2bZtW4oqNMbE0+Lpv6qal8yMRcQL3AvMBSqAdSKyWFVj1xRPqupVMdOeC5wCTAYygeUi8pyqVrnjC4EBydQX7V+330791s7tjyRz3FiO/H7LN0e2/kg6pz+SrVu3Mn36dLKzswE444wzWLRoEddff307/lrGmGSksofEqcB2VX1PVRtw+i/5fILTjgeWq2pQVQPAJmAeRALqp0CPXlNYfySd0x/JhAkTWLFiBZWVldTW1rJkyRI++uijJP86xpj2SOSCxI46Coj+RlcA0+K0O19EZgHvAN9V1Y9wguNmEfk5kA3MAZq2ZK4CFqvqJ63dfrw9Wtty6ArWH0nH+yMZN24cN9xwA3PnziU3N5fPfOYz+Hyp/G9tjImVym9cvLV87EH6Z4HHVbVeRL4BPAKcqaqlInIqsBrYA7wKBEVkOPBlYHabby5yBXAFwMiRIzv8IbqC9UeSXH8kX/va1/ja174GwPe//31GjBjR4vyMMZ0vlbu2KoCjo16PAHZGN1DVSlVtulHU74EpUeN+oqqTVXUuTihtA04Gjge2i8gOIFtEtsd7c1W9X1ULVbWwoKCgsz5Tp7H+SDqvP5Ldu3cD8OGHH/KXv/zF+iUxpoulcotkHTBGRI4FPgYuBC6ObiAiw1T1E/flfGCrO9wLDFDVShGZBEwCSlU1CBwZNX2Nqh6fws+QMtYfSef1R3L++edTWVmJ3+/n3nvvZeDAgR1eJsaY9muzP5KkZi5yDnAP4AUeVNWfiMitwHpVXSwid+AESBDYC3xTVd8WkSxgozubKuAbqloeZ/41qprbVh3WH0n3Yv2RGNMzJNofSUqPSqrqEmBJzLCbop7fCNwYZ7qDOGdutTX/NkPEGGNMatnpLT2c9UdijEm3Ph0kqtrqGUM9gfVHkphU7sI1pq9L5Vlb3VpWVhaVlZW2gukDVJXKyspmpzcbYzpPn90iGTFiBBUVFezZsyfdpZgukJWVZdeXGJMifTZI/H4/xx57bLrLMMaYHq/P7toyxhjTOSxIjDHGJMWCxBhjTFIsSIwxxiTFgsQYY0xSLEiMMcYkxYLEGGNMUixIjDHGJMWCxBhjTFIsSIwxxiTFgsQYY0xSLEiMMcYkxYLEGGNMUixIjDHGJCWlQSIi80TknyKyXUS+F2f8QhHZIyLl7uPyqHF3ichm93FB1PBH3XluFpEHRcSfys9gjDGmdSkLEhHxAvcCZwPjgYtEZHycpk+q6mT38YA77bnAKcBkYBpwnYjku+0fBcYCE4F+wOVx5mmMMaaLpHKLZCqwXVXfU9UG4Ang8wlOOx5YrqpBVQ0Am4B5AKq6RF3AWsC6vTPGmDRKZZAcBXwU9brCHRbrfBF5Q0SeFpGj3WGbgLNFJFtEBgNzgKOjJ3J3aV0KPN/5pRtjjElUKoNE4gzTmNfPAqNUdRLwAvAIgKqWAkuA1cDjwKtAMGba3wArVLUs7puLXCEi60VkvfXLbowxqZPKIKmg+VbECGBndANVrVTVevfl74EpUeN+4h43mYsTStuaxonIzUAB8F8tvbmq3q+qhapaWFBQkPSHMcYYE18qg2QdMEZEjhWRDOBCYHF0AxEZFvVyPrDVHe4VkUHu80nAJKDUfX05UAJcpKrhFNZvjDEmAb5UzVhVgyJyFbAU8AIPqupbInIrsF5VFwNXi8h8nN1We4GF7uR+oExEAKqAS1S1adfW74APgFfd8X9R1VtT9TmMMca0TpyTn3q3wsJCXb9+fbrLMMaYHkVENqhqYVvt7Mp2Y4wxSbEgMcYYkxQLEmOMMUmxIDHGGJMUCxJjjDFJsSAxxhiTFAsSY4wxSbEgMcYYkxQLEmOMMUmxIDHGGJMUCxJjjDFJsSAxxhiTFAsSY4wxSbEgMcYYkxQLEmOMMUmxIDHGGJMUCxJjjDFJsSAxxhiTFAsSY4wxSbEgMcYYk5SUBomIzBORf4rIdhH5XpzxC0Vkj4iUu4/Lo8bdJSKb3ccFUcOPFZHXRGSbiDwpIhmp/AzGGGNal7IgEREvcC9wNjAeuEhExsdp+qSqTnYfD7jTngucAkwGpgHXiUi+2/4u4G5VHQPsA76Wqs9gjDGmbancIpkKbFfV91S1AXgC+HyC044HlqtqUFUDwCZgnogIcCbwtNvuEeALnVy3McaYdkhlkBwFfBT1usIdFut8EXlDRJ4WkaPdYZuAs0UkW0QGA3OAo4FBwH5VDbYxT0TkChFZLyLr9+zZ0xmfxxhjTBypDBKJM0xjXj8LjFLVScALOFsYqGopsARYDTwOvAoEE5wn7jzuV9VCVS0sKCjo2CcwxhjTplQGSQXOVkSTEcDO6AaqWqmq9e7L3wNTosb9xD1uMhcnQLYBnwIDRMTX0jyNMcZ0rVQGyTpgjHuWVQZwIbA4uoGIDIt6OR/Y6g73isgg9/kkYBJQqqoKvAx8yZ3mK8DfUvgZjDHGtMHXdpOOUdWgiFwFLAW8wIOq+paI3AqsV9XFwNUiMh9nt9VeYKE7uR8oc46tUwVcEnVc5AbgCRG5DXgd+EOqPoMxSVOFhhoIfOo8aj+FwB73UQkN1eDxtfzwtjKurfFeP3i8UcNiXnv97nNv1HgfeOzyMtM+4vzI790KCwt1/fr16S7D9BaNdW4w7IkfDk3Pa93nwYPx5+PPgcw80BCEgxAKOv+GgxBu7NrP1IzECRp/VCB5Wwkhbzum7cD4ZIL1sHC14GyLiGxQ1cK22qVsi8SYHiPY4IZBvHCIGt7UpqEm/ny8mZBTADmDnceQcc6/2YPd4QWQM8j5N3swZGS3Xlc4FBUssUET5xEbROHgoXmEYl63Z3xkXNP4mNex44MHW6iptWm7W3C2FI4tbOG1K1g7M3gT2CrN6u9Ml0IWJKb3CQWhbm9UMERtHcQLh4MH4s/H44sKgcFwxLFuCAyKCgY3NHIKICMXJN6JhR3k8borgMzOm2d3Fg63EVKtBWdT+EWFbyjmdfT4zgzW6OBsNr61eXdhcH5rHRSckNK3sCAx3V84DAf3R4VAbDjEBEPtXuKeFS4e6HfEoQAYNulQIMQLh6wBzYJBVQlpyHmEQwQ1SCjsvA4GA4Qaqw4bfli7cPDQv+EQCOT4c8j155Ljz4k87+frh3RmKPUEHg94MrHgbM9WZwLBmZv6yx8sSEy7hDXc4soxFHaeNxsXr10oSKihmmDdPkJ1+wkd3E/w4H5C9VWE6qsI1lcRaqghVF9DqKGGYGMtIZSQQBAhJBBCCAqEfFmE/FmEfP0IDuhHaNBoQr5xBL0ZhLw+Qh4fQY+XoMdLSIRQs/prCDUeILj3bUKVh6/0D3utoS5bzoJEgqUpXLL92c0CJ3pcTkYOOb4ccjMOb5flzep7odQT9KLgtCBpJ1VtcSV62Eq1pZVtAr9Y463EYsc1/art6PxbXNG3Mk+Nf/1namQCmX6gf2SQB8ErXnweH16PD6/H67yWqOceD15RvBLGSwgfgled6TI8GXh9TjuvJ3Y6X2R47Ou47dqYR+zweOPCGibQGGj2qGmsOXxYQw2BYIA9tXsIBAMEGpx2ifw9vOI9LFziBVPcYRm55PhynKDy55DhybBQMoexIGnFtcuvZfXHqyMr0aAGCWs43WU1W0F1ZGWYIRl4PV784m95PoA31Ig3WI8v2IA3WI+3sQ5v8CC+hlq8jbV4G2rx1QfwNlTjDTXgU+c8b68qXsCniteTgTerP77MfLxZA/D2G4Cv3wC8/Y5wH4PwZQ/CmzMIb/ZgfP6cVmv3iJ1h00RVqQvWtRpENY011DbWHjasqqGKnYGdBBoCTjA1BhJ6T5/HF3erKHb3XCLD/F5/ipeQ6SoWJK2YeuRUCvoVJPwrtc1fsC3NI840Lc3DI56O/SIMNcYccI4+TfXT5scZAp861zfE4804dBwhezgMjDngHH1wOmcwZOQk90cwLRIRsv3ZZPuzKSC5/eBhDVMXrIts+TRt8cQLodjg2ntwLxXVFZFxdcG6hN4zw5Ph7IrzZTtbPq2EUGtbUDn+HHweW5Wlky39Viw4cUG6S2hZOAR1+1o+Eyl22MH98ecj3qgQGARHTWl+mmokHNw2mXmde2aS6RY84omsoJMVCocIBANOCEUFUyDo7qJrZQtqT+0edjTuiAyrD9W3/YZAljcrsisuEkzuLrlEd+Pl+HPI9mXjTfGpsr2RBUl3oeoEQ2unqTa7zqGS+PerFMg+4lAIDD3p8GsYosMha4BdkGU6ldfjJT8jn/yMfEgylxrDjdQ21ra8y66VLahPAp9Qs//QsMYET7nN9mW3vXsugS2ovnTmnQVJqqhCfXWc01RbuOCt9lPnlL14svofCoBBx8PI0w6/hqFpl1L2ESm/+MiYruL3+Omf2Z/+mf3bbtyGhlBDi8eNWtp1V9NYQ6AhwN76vYeOJzUECGoL39UorZ1519oWUk88886CpD0aauNvHbQUDi1tlmfkHdo6GDAShp8c/wK37MHO7iaf9SZsTLIyvBlkeDMYmDUwqfmoKvWh+rbPtGvhGNOndZ9GAioQDCR0Ak+iZ97F2zoaP2g82f427qKQJAuS1rzwI3h/xaGD0y2d2eLLgpwhTjjkDoUhJx0Kg+h/m3Yn+ft16ccwxnQeESHLl0WWL4tB/QYlNa+mM+9qg7XNjh8lsrUUOfPOfV3bWBv3dPC/feFvHNf/uKTqbIsFSWvCIWe30qDjmx9wjg2HjBw7AG2MabfoM+8G9xuc1LzinXkXCAYYljOs7YmTZEHSmuIfp7sCY4xJSGeeedfu9+7ydzTGGNOrWJAYY4xJigWJMcaYpFiQGGOMSYoFiTHGmKRYkBhjjEmKBYkxxpikWJAYY4xJiqh2YY93aSIie4APOjj5YODTTiyns1hd7WN1tY/V1T69ta5jVLXNzm76RJAkQ0TWq2phuuuIZXW1j9XVPlZX+/T1umzXljHGmKRYkBhjjEmKBUnb7k93AS2wutrH6mofq6t9+nRddozEGGNMUmyLxBhjTFIsSAAReVBEdovI5hbGi4j8UkS2i8gbInJKN6lrtogcEJFy93FTF9V1tIi8LCJbReQtEflOnDZdvswSrKvLl5mIZInIWhHZ5NZ1S5w2mSLypLu8XhORUd2kroUisidqeV2e6rqi3tsrIq+LyN/jjOvy5ZVgXWlZXiKyQ0TedN9zfZzxqf0+qmqffwCzgFOAzS2MPwd4DhBgOvBaN6lrNvD3NCyvYcAp7vM84B1gfLqXWYJ1dfkyc5dBrvvcD7wGTI9pcyXwO/f5hcCT3aSuhcCvu/r/mPve/wU8Fu/vlY7llWBdaVlewA5gcCvjU/p9tC0SQFVXAHtbafJ54P/UsQYYICIp778ygbrSQlU/UdWN7vNqYCtwVEyzLl9mCdbV5dxlUOO+9LuP2IOTnwcecZ8/DXxWJLX9NydYV1qIyAjgXOCBFpp0+fJKsK7uKqXfRwuSxBwFfBT1uoJusIJynebumnhORE7q6jd3dymcjPNrNlpal1krdUEalpm7O6Qc2A0sU9UWl5eqBoEDwKBuUBfA+e7ukKdF5OhU1+S6B7geCLcwPi3LK4G6ID3LS4FSEdkgIlfEGZ/S76MFSWLi/dLpDr/cNuLcwuAzwK+Av3blm4tILvAMcI2qVsWOjjNJlyyzNupKyzJT1ZCqTgZGAFNFZEJMk7QsrwTqehYYpaqTgBc4tBWQMiLyOWC3qm5orVmcYSldXgnW1eXLyzVDVU8Bzga+JSKzYsandHlZkCSmAoj+ZTEC2JmmWiJUtapp14SqLgH8IjK4K95bRPw4K+tHVfUvcZqkZZm1VVc6l5n7nvuBV4B5MaMiy0tEfEB/unC3Zkt1qWqlqta7L38PTOmCcmYA80VkB/AEcKaI/CmmTTqWV5t1pWl5oao73X93A4uAqTFNUvp9tCBJzGLgP9wzH6YDB1T1k3QXJSJHNu0XFpGpOH/Pyi54XwH+AGxV1Z+30KzLl1kidaVjmYlIgYgMcJ/3A84C3o5pthj4ivv8S8BL6h4lTWddMfvR5+Mcd0opVb1RVUeo6iicA+kvqeolMc26fHklUlc6lpeI5IhIXtNzoBiIPdMzpd9HX2fNqCcTkcdxzuYZLCIVwM04Bx5R1d8BS3DOetgO1AKXdZO6vgR8U0SCQB1wYaq/TK4ZwKXAm+7+dYDvAyOjakvHMkukrnQss2HAIyLixQmuP6vq30XkVmC9qi7GCcA/ish2nF/WF6a4pkTrulpE5gNBt66FXVBXXN1geSVSVzqW11Bgkfv7yAc8pqrPi8g3oGu+j3ZluzHGmKTYri1jjDFJsSAxxhiTFAsSY4wxSbEgMcYYkxQLEmOMMUmxIDHGGJMUCxJjugn3VuAdusrevX358M6YlzHtZUFiTO+wEBjeViNjUsGCxJgYIjJKRN4WkQdEZLOIPCoiZ4nIKhHZJiJT3cdqcTo4Wi0iJ7rT/peIPOg+n+hOn93C+wwSkVJ3HvcRdWM9EblEnE6nykXkPvfqc0SkRkR+JiIbReRF9zYnXwIKgUfd9v3c2XzbbfemiIxN5TIzfZsFiTHxHQ/8ApgEjAUuBmYC1+LcduVtYJaqngzcBNzuTncPcLyInAc8BPynqta28B43AyvdeSzGvZWLiIwDLsC5o+tkIAT8uztNDrDRvdPrcuBmVX0aWA/8u6pOVtU6t+2nbrvfunUbkxJ2ry1j4ntfVd8EEJG3gBdVVUXkTWAUzt1mHxGRMTi34266B1pYRBYCbwD3qeqqVt5jFvBFd7p/iMg+d/hnce4au869f1I/nP5CwOkH40n3+Z+AeHdebtI0bkPT+xiTChYkxsRXH/U8HPU6jPO9+THwsqqeJ04nWq9EtR8D1JDYMYt4N7sT4BFVvbGD0zdpqjmEfddNCtmuLWM6pj/wsft8YdNAEemPs0tsFjDIPX7RkhW4u6xE5GxgoDv8ReBLIjLEHXeEiBzjjvPg3MEYnN1tK93n1Tj91BvT5SxIjOmY/wHuEJFVgDdq+N3Ab1T1HeBrwJ1NgRDHLcAsEdmI04fEhwCqugX4IU7XqW8Ay3Bu+Q4QAE4SkQ3AmcCt7vCHgd/FHGw3pkvYbeSN6UFEpEZVc9NdhzHRbIvEGGNMUmyLxJgUE5HLgO/EDF6lqt9KRz3GdDYLEmOMMUmxXVvGGGOSYkFijDEmKRYkxhhjkmJBYowxJikWJMYYY5Ly/wGgvlpWeNx8oQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2adb6d8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "（2）微调，步长减小"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 5, 6], 'min_child_weight': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = [4,5,6]\n",
    "min_child_weight = [0.5,1,2]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59342, std: 0.00334, params: {'max_depth': 4, 'min_child_weight': 0.5},\n",
       "  mean: -0.59336, std: 0.00317, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.59340, std: 0.00355, params: {'max_depth': 4, 'min_child_weight': 2},\n",
       "  mean: -0.58928, std: 0.00408, params: {'max_depth': 5, 'min_child_weight': 0.5},\n",
       "  mean: -0.58876, std: 0.00418, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58980, std: 0.00301, params: {'max_depth': 5, 'min_child_weight': 2},\n",
       "  mean: -0.58922, std: 0.00361, params: {'max_depth': 6, 'min_child_weight': 0.5},\n",
       "  mean: -0.58936, std: 0.00340, params: {'max_depth': 6, 'min_child_weight': 1},\n",
       "  mean: -0.58912, std: 0.00383, params: {'max_depth': 6, 'min_child_weight': 2}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.5887620837531603)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([233.5954    , 252.27488012, 299.66128001, 327.38860011,\n",
       "        303.64003997, 291.7809999 , 363.10633998, 441.63498006,\n",
       "        304.40299997]),\n",
       " 'mean_score_time': array([0.55660005, 0.59639993, 1.01079998, 0.7204    , 0.72780004,\n",
       "        0.68660002, 1.40120001, 1.96919999, 0.85520005]),\n",
       " 'mean_test_score': array([-0.59341847, -0.59335921, -0.59339942, -0.58927755, -0.58876208,\n",
       "        -0.58980421, -0.58922418, -0.58936275, -0.58911558]),\n",
       " 'mean_train_score': array([-0.54937886, -0.54996551, -0.55122232, -0.51208093, -0.51338803,\n",
       "        -0.51633909, -0.46402847, -0.46703564, -0.4735754 ]),\n",
       " 'param_max_depth': masked_array(data=[4, 4, 4, 5, 5, 5, 6, 6, 6],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[0.5, 1, 2, 0.5, 1, 2, 0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 4, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 4, 'min_child_weight': 1},\n",
       "  {'max_depth': 4, 'min_child_weight': 2},\n",
       "  {'max_depth': 5, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 2},\n",
       "  {'max_depth': 6, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 6, 'min_child_weight': 1},\n",
       "  {'max_depth': 6, 'min_child_weight': 2}],\n",
       " 'rank_test_score': array([9, 7, 8, 4, 1, 6, 3, 5, 2]),\n",
       " 'split0_test_score': array([-0.58780509, -0.58756265, -0.58780986, -0.58274125, -0.58164061,\n",
       "        -0.58505136, -0.5827374 , -0.58324898, -0.58306835]),\n",
       " 'split0_train_score': array([-0.55158826, -0.55202564, -0.5531848 , -0.51380374, -0.5146005 ,\n",
       "        -0.51872499, -0.46417406, -0.46672137, -0.47564157]),\n",
       " 'split1_test_score': array([-0.59266726, -0.59348132, -0.59200251, -0.58737787, -0.58687398,\n",
       "        -0.58799014, -0.58875776, -0.59072817, -0.58730832]),\n",
       " 'split1_train_score': array([-0.54909552, -0.55026426, -0.55097063, -0.51182202, -0.51360895,\n",
       "        -0.51707194, -0.46363255, -0.46719345, -0.47258462]),\n",
       " 'split2_test_score': array([-0.59306515, -0.59345623, -0.59273958, -0.58918546, -0.58986775,\n",
       "        -0.59016758, -0.59049362, -0.58930141, -0.59149147]),\n",
       " 'split2_train_score': array([-0.54924092, -0.54978611, -0.55087859, -0.51237954, -0.51342309,\n",
       "        -0.51587207, -0.46426129, -0.46756419, -0.4730217 ]),\n",
       " 'split3_test_score': array([-0.59742643, -0.59687039, -0.59720529, -0.59312065, -0.5922439 ,\n",
       "        -0.59273655, -0.59044723, -0.58988941, -0.58933878]),\n",
       " 'split3_train_score': array([-0.54858872, -0.5488195 , -0.55055786, -0.51225492, -0.51396996,\n",
       "        -0.51569399, -0.4639109 , -0.46661702, -0.4730036 ]),\n",
       " 'split4_test_score': array([-0.59612923, -0.59542608, -0.59724102, -0.59396393, -0.59318552,\n",
       "        -0.59307641, -0.59368626, -0.5936471 , -0.59437258]),\n",
       " 'split4_train_score': array([-0.54838085, -0.54893205, -0.55051971, -0.51014441, -0.51133766,\n",
       "        -0.51433245, -0.46416354, -0.46708214, -0.47362552]),\n",
       " 'std_fit_time': array([ 9.07457951, 20.02641281, 41.76797919, 35.58593933,  9.10015307,\n",
       "        11.99734241, 22.01589804, 29.13704441, 46.69316459]),\n",
       " 'std_score_time': array([0.04134541, 0.05372736, 0.55836026, 0.02857706, 0.04693152,\n",
       "        0.01591977, 0.38464656, 0.83992422, 0.10543882]),\n",
       " 'std_test_score': array([0.00333503, 0.00316943, 0.00354603, 0.00407604, 0.00417593,\n",
       "        0.00301058, 0.00361417, 0.00340227, 0.00382523]),\n",
       " 'std_train_score': array([0.00114901, 0.00116163, 0.00099679, 0.00117513, 0.00110116,\n",
       "        0.00147584, 0.00022988, 0.00034072, 0.00108506])}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_2.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588762 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAELCAYAAADgPECFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmYVNW97vHvrwe6kZlmkBaQ2eCAQNoBFRxBTQxqJFEyHIgh5tx444058YhJjkOiucRDovHJiSYxURMTc4yJJ8SLCgJKHJA0BByQMInYggjN2EA3PfzuH3tXV3VTPVR3765qeD/PU0/V3nvtXauLot5aa+9ay9wdERGRqGSluwIiInJ0U9CIiEikFDQiIhIpBY2IiERKQSMiIpFS0IiISKQUNCIiEikFjYiIREpBIyIikcpJdwUyQZ8+fXzIkCHproaISIeyYsWKne7et6lyChpgyJAhFBcXp7saIiIdipm915xy6joTEZFIKWhERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZR+RyOSaWqqYW8J7NoE+z6AAadD/1PBLN01E2kRBY1IOtTUBCGyayOUbgxCZdem4PHud6H6cN3y3Qph5GQYOQWGXQB5XdNRa5EWUdCIRKWmBvZvSwiTjVAaBsrud6GqPF42Jx96DYU+I2HUpVAwHHoPh27Hw/uvw/oF8PbTsPIxyO4EJ54ThM7IsKxaO5LBzN3TXYe0Kyoqcg1BIy3iHobJpoQwibVQ3oWqQ/Gy2XnQeyj0HhbcYmFSMDxosWQ1ccq0uhK2LIP1z8P6hbBjbbC+19AgnEZOhhPPg9z86P5ekQRmtsLdi5osp6BR0EgT3KFse0KA1OvuqjwYL5vdCXoNiQdI76Hxx91PgKzstqvX7veCls76hfDu0iDUco+DoefHu9l6Dmq75xOpR0GTAgWN4A4HdiRplWwMWiaHy+Jls3LqhUlCC6XHoLYNk+aqPASbX4Z1zwctnj1bgvX9Tg672KbAoDMhO7f96yZHLQVNChQ0xwh3OFhaL0zCQCndBIf3x8tm5UDPE+t1cQ0L7nsMguwMPr3pDjvXh11sC+C9V6GmCvJ6wIiLgtAZMRm6Njm6u0ijmhs0Gfy/RaQF3OHgrnpdXLH7d6Fib7ysZUPPwUGQDDo7IVSGBes76rd/M+g7Krid83Uo3webXox3s739NGBQOC5+bmfAuKbPEYm0UFpaNGbWG/hvYAiwGfisu+9OUq4aeDNc3OLuU8P1FwP/SfCD0zJgprtvMLMTgV8DfYFdwBfcvaSp+qhF0wHFwuSI8yYboTwxTLKCFkjiiffeYcuk52DI6ZS+vyEd3OHDN2DdgiB4Sv4OOHTpG7RyRk6G4RdB557prql0ABnddWZm9wK73H2Omc0Gern7rUnKlbn7ET8YMLN1wJXu/o6ZfQ04091nmtkfgWfc/TEzuwj4krt/san6KGgy1KE9CZcEb6x7ZdehxO8lFobJsIQwCe97DoacvLT9CRnvQClsXBSEzoYXgtfVsmHw2eEFBZdCv9G6fFqSyvSg+SdwgbtvM7MBwIvuflKScg0FzT+Bf3H3183sNqCbu3/bzN4GLnX3EjMzYK+7d2+qPgqaNCrfW7dlknje5GBpQkGDHgOTXxrca4jCpC3UVENJcfzczodhZ0KPQfGr2IZOgk5d0ltPyRiZHjR73L1nwvJud++VpFwVsAqoAua4+/+E6ycC/wMcAvYBZ7v7PjP7PfC6u//EzD4N/Ano4+6lSY59A3ADwODBgz/+3nvNmpFUWqJif70uroSuroM765btfkLdcyWJYZLbOS3VP2bt2xqc01m/IDjHc7gs+C3QkPOC0Bk1Jfg3kmNW2oPGzF4Ajk+y6TvAY80MmkJ332pmw4DFwMXuvtHM/gz8MGzR3AKc5O6zzKwQ+CkwFFgKXAOc4u576x87kVo0baCirN65knfjjw98VLdstwF1r+KKhUqvodDpuPTUXxpXVQFbXouf2yldH6wvGBF0r42cHIxWoJblMSXtQdPokzaz66zePo8CzwAvAcvcfXi4fjDwnLufXK98V2Ctuw9sqj4KmmY6fPDIE++73g0el31Yt2zX44/8wWLvcFldLx3frk3x1s67f4PqCujUNRiHLdbN1r0w3bWUiGX65c3zgBnAnPD+L/ULmFkv4KC7V5hZH+Bc4F5gN9DDzEa5+zpgMvBOuE8fgosMaoDbCK5Ak1RUHqrbGkns6tq/rW7ZLv2CABlxSRAgtWEyTIM+Hu16D4OzvhrcDh8IwiY2NM7aZ4Iy/U8LutdGToGBZ6Tnh6ySEdLVoikAngQGA1uAz7j7LjMrAv417AY7B/g5UENwGfP97v6rcP+rge+F23YD17v7JjObBvxfwAm6zm5094qm6nPMtWgqy4NBHZMNqbLvg7plj+tz5A8WYyfk85u8zkKONe7BGGzrwtDZ8hp4NXTuBcMvDn8segl0KUh3TY8p7s6hqkPsrdjLvsP72Fuxl72H97KvYh8jeo3g9L6nt+i4Gd11lmmOyqCpqoDdm+v9+j2831tCkMWh4wrqnnhPPBmf3yNdf4EcDQ7tgU1LgnM7GxYGw/xgMLAofm7n+DH6sWgzVdVU1QZFbWCEj/dV7KsNj72H99Yps69iH1VelfSYM0+Zyb8V/VuL6qOgSUGHDZqqw7DnveRDquwtAa+Jl+3cK94aqdNCGRZsE4laTQ1sWxWOULAAPlgJOHTtnzDXzoVHfUs5sXXRYDAkBERiC+RA5YFGj901tys98nrQvVN3uud1p0enHrXLPfLqPo7d98zrSX5Oy0b8VtCkIKODproyGKU32ZAqe9+vGyb5PeqdeE9omRzXO31/g0gyZTuCH4mufx42LA6GB8rKgcETwqFxpkCfURn7Y9EoWhcAOVk5DQZELDyShUi3Tt3IyWrf0+4KmhSkPWiqq4KWSeIsi7Ew2bMl6OOOyet+5A8WeyeESYb+pxRpVHUVlCyPn9v56O1gfc/BYRfbFBg6sc1/S5VprYvunbrTOacz1kH+HytoUtDSoFm2bRn3rbiP/Ox8Oud0Jj8nP7glLmeH67JyyT98kPxDe+h8oJT8sh3k7/+Q/D0f0HnfB+RXV5LnTucaJ6dTNyzWrVV/SJXjChQmcvTb835wTmfdAnj3pWDOn5z8YGSC2LQHvU6sLa7WRXpk+uXNR4XcrFwK8gsory5n3+F9bD+4nfLDZZRXHqSiupxDNZVU0USQdwO69a+zKtuyyc85TH7NZvL3fEjnspXklwSBlZeTR+fsxkOtc05n8rLzah/Xhl295WPhP4J0HHVaFzUH2HfCqeztO4i9465k3/Y32bvjbfbu/if7ileyb+W97O3UmX2d8tmLc6C6vNFj129d9D+u/1HXushk+qRphY97Hh+v7hXv6tq9GaoPxwvkHkdl76FU9BpCea/BHOpxAuXd+lPetR/lnY6jvLqC8upyyqvKOVR1iPKq8rrL4eOK6ora7XvL97K9evsR5asTu9eaKScrJx5CCSHVUGg1q+VWL9TysvMUaMeYSFsXPXrTI+cEuldV0b98H6P27qR7dRXdLZcevUfQ/fix9Bh8Dj16DjnmWheZTK9+a+x9H/7+cNDF1WcUnHR53UuEuw0g14xcIOqfL1bWVAbBE94OVR+KL1cHwVVRXdFkqMXW7yrfVWff2OOaxIsPmik3KzcInuzO5OXk1T5ODKxkra86oZVYPmE5LzuvNtCy9YPANtOccxcNhUeq5y5a1bqoKAu61mLndta/BjwIA06Pn9s5YXx0L5Q0i87R0IqLAaqrgvlOjpHfALg7lTWVR4ZWdfKAS2yJNdZyix0rMdS8qS7HJDpldUraRZgsvBrtbmwg1GLBlmUd5987sXVR/wT3UXfuwh22vxVcOr1uQXBxgdcE5zVHXBKEzvCLdAVmG9LFAClI+1VnUoe7c7jmcJ1gqg2txFBLEnIVVRV1W2FJWm6JZVoiaVdivdZXXk5e87oaG2jR5WXn1X57b8/WxVF1ZdTBXbBxcXyunYOlwRfDgWfGh8bpf6ourmkFBU0KFDTHJndP2pqq7WqsqmiwezExtOp3UyY7x9YSnXM6k5uVy8HKgx2vdZFpaqqDH4iuXxD8bmfb6mB9t8KEH4teoDH6UqSgSYGCRqJU4zXxQKt//ixhOVlXY3lVOV1yu3T81kWm2f9h0MpZ9zxsXAKH90N2p2Cqg9i5nT4j0l3LjKegSYGCRuQYVnUY3n89Pvr0jrXB+t7Dwt/sTIYTz4Pclg3TcjRT0KRAQSMitXZvTphrZylUlUPucTD0/Hg3W89B6a5lRtAPNkVEWqLXEDjzK8Gt8lA41054bmfds0GZfifHRygYdBZk66O0MWrRoBaNiDSDO+xcF14+/Xww105NFeT1gBEXhXPtTIaufdNd03ajFo2ISFsyg74nBbdzvg7l+4K5dtYvCLra3n4aMCgcF44+PRkGjDtmfmfXGLVoUItGRFqppgY+fCM8t/M8lBQDDl36Bq2ckZODH4t27pnumrYpXQyQAgWNiLSpA6XhXDvhj0XL94Blw+Cz4+d2+o3u8D8WVdCkQEEjIpGproIPiuMzi374ZrC+x6D4VWxDJ0GnLumtZwsoaFKgoBGRdrNva/zy6Y1LoPIAZOfBkPPi53Z6D0t3LZtFQZMCBY2IpEVVBbz3ajx4StcH6wtGhCMUTA5GK8jJS289G6CgSYGCRkQyQunG+NA4m1+G6gro1DUYhy3Wzda9MN21rKXLm0VEOpqCcD6rs74Khw8EIxPEpj1Y+0xQpv9p8dGnB54BHWAeJrVoUItGRDKcO3z0TvyCgi3LwKuhcy8YfnH4Y9FLoEtBu1ZLLRoRkaOFGfQ/Obid9w04tCeca2chbFgIbz0FGAwsip/bGXB6xlw+rRYNatGISAdWUwPb/hGEzrrnYevKYH3X42FkOLPosAshv3ubP7UuBkiBgkZEjhplHyX8WHQxVOyFrBwYPCG8fHoK9BnVJq0dBU0KFDQiclSqroT3l8fP7Xy0Jljf88T4CAVDJ0Ju5xYdXkGTAgWNiBwT9rwfHwT03Zeg8iCc8RX45NwWHU4XA4iISF09B8EZXw5uleXw3svQbUDkT6ugERE5FuXmB5dEtwNNlCAiIpFS0IiISKTSEjRm1tvMFprZ+vC+VwPlqs1sVXibl7D+IjNbaWZvmdljZpYTrjcze8DMNpjZG2Y2vr3+JhERSS5dLZrZwCJ3HwksCpeTOeTuY8PbVAAzywIeA65z91OB94AZYfnLgZHh7QbgwQj/BhERaYZ0Bc2VBGFBeH9VCvsWABXuvi5cXghck3Dc33hgGdDTzKK/pEJERBqUrqDp7+7bAML7fg2UyzezYjNbZmaxMNoJ5JpZ7NrtacCg8PEJwPsJ+5eE60REJE0iu7zZzF4Ajk+y6TspHGawu281s2HAYjN70903mtl1wH1mlgcsAKpiT5vkGEl/kWpmNxB0rzF48OAUqiQiIqmILGjcvcELtM1su5kNcPdtYdfWRw0cY2t4v8nMXgTGARvd/TVgYnisKcCocJcS4q0bgIHA1gaO/QvgFxCMDJDCnyYiIilIV9fZPOIn8GcAf6lfwMx6hS0WzKwPcC6wJlzuF97nAbcCDyUc91/Cq8/OBvbGuuhERCQ90hU0c4DJZrYemBwuY2ZFZvZwWGY0UGxmq4ElwBx3D0eE4xYzewd4A/iruy8O188HNgEbgF8CX2uXv0ZERBqkQTXRoJoiIi3R3EE1NTKAiIhESkEjIiKRUtCIiEikFDQiIhIpBY2IiERKQSMiIpFS0IiISKQUNCIiEikFjYiIREpBIyIikVLQiIhIpBQ0IiISKQWNiIhESkEjIiKRUtCIiEikFDQiIhKpnHRXQETqqqyspKSkhPLy8nRXRQSA/Px8Bg4cSG5ubov2TylozKwXMMjd32jRs4lIk0pKSujWrRtDhgzBzNJdHTnGuTulpaWUlJQwdOjQFh2jya4zM3vRzLqbWW9gNfCImf24Rc8mIk0qLy+noKBAISMZwcwoKChoVQu7Oedoerj7PuDTwCPu/nHgkhY/o4g0SSEjmaS178fmBE2OmQ0APgs806pnExGRY05zguZ7wPPABnf/u5kNA9ZHWy0RSZc9e/bws5/9rEX73n///Rw8eLDVdSguLuamm25q9XFiZs6cyVNPPXXE+q1btzJt2jQAXnzxRa644oqk+w8ZMoSdO3e2WX1iZs2axZo1axot01DdN2/ezO9///sG93vuuec46aSTGDFiBHPmzEla5tFHH6Vv376MHTuWsWPH8vDDD6f2BzRTk0Hj7n909zHu/rVweZO7XxNJbUQk7TIhaIqKinjggQdafZymFBYWJv0Qby8PP/wwJ598cov2bSxoqqurufHGG3n22WdZs2YNTzzxRIOBdu2117Jq1SpWrVrFrFmzWlSXpjR51ZmZ3QvcDRwCngNOB77h7o9HUiMRqXXXX99mzdZ9bXrMkwu7c8enTmlw++zZs9m4cSNjx45l8uTJ9OvXjyeffJKKigquvvpq7rrrLg4cOMBnP/tZSkpKqK6u5j/+4z/Yvn07W7du5cILL6RPnz4sWbIk6fG7du3KjTfeyAsvvECvXr34wQ9+wL//+7+zZcsW7r//fqZOncqLL77I3LlzeeaZZ7jzzjvZsmULmzZtYsuWLXzjG99otLXzm9/8hrlz52JmjBkzht/+9rcALF26lB//+Md8+OGH3HvvvUybNo3NmzdzxRVX8NZbb9U5RmlpKdOnT2fHjh2ceeaZuHuDz3fvvfeSn5/PTTfdxM0338zq1atZvHgxixYt4pFHHuHxxx9nwYIF3HHHHVRUVDB8+HAeeeQRunbtygUXXMDcuXMpKiriV7/6FT/84Q8pLCxk5MiR5OXl8dOf/rTBus+ePZt33nmHsWPHMmPGDG6++ebaOi1fvpwRI0YwbNgwAK677jr+8pe/tDjUWqs5XWdTwosBrgBKgFHALZHWSkTSZs6cOQwfPpxVq1YxefJk1q9fz/Lly1m1ahUrVqxg6dKlPPfccxQWFrJ69WreeustLrvsMm666SYKCwtZsmRJgyEDcODAAS644AJWrFhBt27d+O53v8vChQt5+umnuf3225Pus3btWp5//nmWL1/OXXfdRWVlZdJyb7/9Nvfccw+LFy9m9erV/OQnP6ndtm3bNl5++WWeeeYZZs+e3ehrcNddd3Heeefxj3/8g6lTp7Jly5YGy06aNIm//e1vQNDlV1ZWRmVlJS+//DITJ05k586d3H333bzwwgusXLmSoqIifvzjuhfubt26le9///ssW7aMhQsXsnbt2jrbk9V9zpw5TJw4kVWrVnHzzTezdetWPvGJTwDwwQcfMGjQoNr9Bw4cyAcffJC0/n/6058YM2YM06ZN4/3332/0dWmp5vyOJvYLnU8AT7j7Ll0RI9I+Gmt5tIcFCxawYMECxo0bB0BZWRnr169n4sSJfOtb3+LWW2/liiuuYOLEic0+ZqdOnbjssssAOO2008jLyyM3N5fTTjuNzZs3J93nk5/8JHl5eeTl5dGvXz+2b9/OwIEDjyi3ePFipk2bRp8+fQDo3bt37barrrqKrKwsTj75ZLZv395oHZcuXcqf//zn2ufu1atXg2U//vGPs2LFCvbv309eXh7jx4+nuLiYv/3tbzzwwAMsW7aMNWvWcO655wJw+PBhJkyYUOcYy5cv5/zzz6+t72c+8xnWrVuXUt0LCwuZP38+QNIWWLLP7U996lNMnz6dvLw8HnroIWbMmMHixYsbe2lapDlB81czW0vQdfY1M+sL6CfLIscAd+e2227jq1/96hHbVqxYwfz587ntttuYMmVKg62R+nJzc2s/9LKyssjLy6t9XFVVlXSfWBmA7OzsBsu5e4OX4iYeo7GusJjmfqHOzc1lyJAhPPLII5xzzjmMGTOGJUuWsHHjRkaPHs3GjRuZPHkyTzzxRIPHaKo+qdZ94MCBdVonJSUlFBYWHlGuoKCg9vFXvvIVbr311iaP3RLNuRhgNjABKHL3SuAAcGUktRGRtOvWrRv79+8H4NJLL+XXv/41ZWVlQNAl89FHH7F161aOO+44vvCFL/Ctb32LlStXHrFvOlx88cU8+eSTlJaWArBr164WHWfSpEn87ne/A+DZZ59l9+7dTZafO3cukyZNYuLEiTz00EOMHTsWM+Pss8/mlVdeYcOGDQAcPHiwTmsF4Mwzz+Sll15i9+7dVFVV8ac//anJOjb2Wp9xxhmsX7+ed999l8OHD/OHP/yBqVOnHlFu27ZttY/nzZvH6NGjm3zelmjOxQC5wBeBSWHCvwQ8FEltRCTtCgoKOPfcczn11FO5/PLL+dznPlfb1dO1a1cef/xxNmzYwC233EJWVha5ubk8+OCDANxwww1cfvnlDBgwoNHzNFE55ZRT+M53vsP5559PdnY248aN49FHH035OHfccQfTp09n/PjxnH/++QwePLjR8hMnTuSee+5hwoQJdOnShfz8/NruxL59+/Loo48yffp0KioqALj77rsZNWpU7f4nnHAC3/72tznrrLMoLCzk5JNPpkePHo0+55gxY8jJyeH0009n5syZXHvttcyaNYv58+eTk5PDT3/6Uy699FKqq6u5/vrrOeWUoBv29ttvp6ioiKlTp/LAAw8wb948cnJy6N27d4teq+awppphZvYwwXmax8JVXwSq3T2a6+DSoKioyIuLi9NdDREA3nnnnci+WUrmKisro2vXrlRVVXH11Vdz/fXXc/XVV6e7WrWSvS/NbIW7FzW1b3PO0Zzh7qcnLC82s9Up1lFERBpx55138sILL1BeXs6UKVO46qqr0l2lNtOcoKk2s+HuvhEgHBmgOtpqiUhHd9ZZZ9V2FcX89re/5bTTTmv1sUtLS7n44ouPWL9o0aI6J7jbUtTPOXfu3FYfI1M1J2huAZaY2SbAgBOBL0VaKxHp8F5//fXIjl1QUMCqVasiO36mPOfRosmgcfdFZjYSOIkgaNYCY6OumIiIHB2aNZWzu1e4+xvuvtrdK4A/tuZJzay3mS00s/XhfdJfQ5lZtZmtCm/zEtZfZGYrzewtM3vMzHLC9R8zs9fMrMLMvtWaOoqISNtoVtAk0dqhAWYDi9x9JLAoXE7mkLuPDW9TAcwsi+AKuOvc/VTgPWBGWH4XcBNw9HZ2ioh0MC0NmqZ/mtq4K4lfLv0YkMrlFQVAhbvHfvG0ELgGwN0/cve/A8kHQhIRkXbXYNCY2V/NbF6S218JPuxbo7+7bwMI7/s1UC7fzIrNbJmZxcJoJ5BrZrFrt6cBg5Lv3jAzuyE8dvGOHTtS3V3kqJUJ0wRoPpq4ls5Hc/3119OvXz9OPfXUVteztRpr0cwFfpTkNpdggM1GmdkL4TmU+rdUhq8ZHP4Y6HPA/eFl1g5cB9xnZsuB/UDygY8a4e6/cPcidy/q27dvqruLHLUyIWg0H03TmgqamTNn8txzz7W0am2qwavO3P2l1hzY3S9paJuZbTezAe6+LZwm+qMGjrE1vN9kZi8C44CN7v4aMDE81hSCqQtEjj7PzoYP32zbYx5/GlyefMZF0Hw00PHno4Fg/LWGRsNuby09R9Na84ifwJ8B/KV+ATPrZWZ54eM+wLnAmnC5X3ifB9yKxl4TaTOaj6bjz0eTaZrzg80ozAGeNLMvA1uAzwCE513+NRxHbTTwczOrIQjEOe4e68y8xcyuCNc/6O6Lw/2PB4qB7kCNmX0DODmcuE2k42mk5dEeNB9Nx5yPJtOkJWjcvRQ4YiwHdy8GZoWPXwWSjlXh7reQZJZPd/8QOPLdJyItovlompaJ89Fkmia7zhq4+uy3ZvZ/zCy/PSopIu1H89F0/PloMk1zztFsAsqAX4a3fcB2ghPwv4yuaiKSDonz0SxcuLB2PprTTjuNadOmsX//ft58803OPPNMxo4dyz333MN3v/tdID4fzYUXXpiWuifOR3P66afzzW9+s0XHueOOO1i6dCnjx49nwYIFzZqPZtu2bUyYMIH+/fs3OB/NmDFjOPvss484B5M4H80ll1yS8nw099133xHnaKZPn86ECRP45z//ycCBA/nVr37VoteiLTRnPpql7j4p2Toze9vd0zupeRvQfDSSSTQfzbHpaJ6Ppjktmr5mVhvn4eM+4eLhVCoqIiLJ3XnnnYwdO5ZTTz2VoUOHHnPz0fwb8LKZbSQY42wo8DUz60J8GBkRkTo0H01qjun5aNx9fjhNwMcIpwlw9/Jw8/1RVk5EOi7NRyMxTQaNmeUCXwVi52leNLOfu7sGrhQRkSY1p+vsQSAXiA1+9MVw3ayoKiUiIkeP5gTNGe5+esLyYjNbHVWFRETk6NKcq86qzWx4bMHMhgHV0VVJRESOJs0JmluAJWb2opm9BCwmuBJNRI5CmTBNgOajiWvJfDTvv/8+F154IaNHj+aUU06pM7hoOjQZNO6+CBhJMEXyTcBJQOvfSSKSkTIhaDQfTdMaC5qcnBx+9KMf8c4777Bs2TL+67/+q8lAi1KzBtV09wrgjdiymf0RaHxMBhFptR8u/yFrd61tumAKPtb7Y9x65q0Nbtd8NB1/PpoBAwYwYMAAIBgTbfTo0XzwwQctDrXWaul8NM0b1lREOhzNR3N0zUezefNm/vGPf3DWWWc1+jdHqaXTBHS8capFOqDGWh7tQfPRdOz5aMrKyrjmmmu4//776d69e6N/c5QaDBoz+yvJA8WAaMZ4EJGMovlompap89FUVlZyzTXX8PnPf55Pf/rTzdonKo11nc0FfpTkNhfIzPlCRaTVNB9Nx5+Pxt358pe/zOjRo1s8VUJbarBF4+4vtWdFRCQzJM5Hc/nll9fORwPBifzHH3+cDRs2cMstt5CVlUVubi4PPvggEJ+PZsCAAY2ep4lK4nw02dnZjBs3jkcffTTl49xxxx1Mnz6d8ePHc/755zdrPpp77rmHCRMm0KVLlwbno4kNMnr33XczatSo2v0T56MpLCxMeT6amTNncu211zJr1izmz5/PK6+8UjuA6dixYwH4wQ9+kPQcTntocj6aY4Hmo5FMovlojk3H+nw0IiISsWN9PppaZna8u38YVWVE5OgAGx6eAAAQKUlEQVSh+WhSc0zPR1PPfGB8FBURkaOL5qORmFS7zvRDTZF2oHOnkkla+35MNWh+2apnE5Em5efnU1paqrCRjODulJaWkp+f3+JjpNR15u4tG2lPRJpt4MCBlJSUsGPHjnRXRQQIvvwkG4mhuVo6BI2IRCQ3N5ehQ4emuxoibUaXN4uISKQUNCIiEikFjYiIREpBIyIikVLQiIhIpBQ0IiISKQWNiIhEKi1BY2a9zWyhma0P75POk2pm1Wa2KrzNS1h/kZmtNLO3zOwxM8sJ13/ezN4Ib6+a2ent9TeJiEhy6WrRzAYWuftIYFG4nMwhdx8b3qYCmFkW8BhwnbufCrwHzAjLvwuc7+5jgO8Dv4jyjxARkaalK2iuJAgLwvtUJl4oACrcPTYX6kLgGgB3f9XdY3OuLgNaPmaCiIi0iXQFTX933wYQ3vdroFy+mRWb2TIzi4XRTiDXzGKzuk0DBiXZ98vAs21ZaRERSV1kY52Z2QvA8Uk2fSeFwwx2961mNgxYbGZvuvtGM7sOuM/M8oAFQFW9576QIGjOa6R+NwA3AE3OBy4iIi0XWdC4+yUNbTOz7WY2wN23mdkA4KMGjrE1vN9kZi8C44CN7v4aMDE81hRgVMKxxwAPA5e7e2kj9fsF4TmcoqIijccuIhKRdHWdzSN+An8G8Jf6BcysV9hiwcz6AOcCa8LlfuF9HnAr8FC4PBj4M/DFhHM4IiKSRukKmjnAZDNbD0wOlzGzIjN7OCwzGig2s9XAEmCOu68Jt91iZu8AbwB/dffF4frbCS4W+Fl4SXRxO/09IiLSANMsfkHXWXGxMklEJBVmtsLdi5oqp5EBREQkUgoaERGJlIJGREQipaAREZFIKWhERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaAREZFIKWhERCRSChoREYmUgkZERCKloBERkUgpaEREJFIKGhERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaAREZFIKWhERCRSChoREYmUgkZERCKloBERkUgpaEREJFJpCRoz621mC81sfXjfq4Fy1Wa2KrzNS1h/kZmtNLO3zOwxM8sJ119pZm+E5YvN7Lz2+ptERCS5dLVoZgOL3H0ksChcTuaQu48Nb1MBzCwLeAy4zt1PBd4DZoTlFwGnu/tY4Hrg4Sj/CBERaVq6guZKgrAgvL8qhX0LgAp3XxcuLwSuAXD3Mnf3cH0XwJPsLyIi7ShdQdPf3bcBhPf9GiiXH3aBLTOzWBjtBHLNrChcngYMiu1gZleb2Vrg/xG0akREJI1yojqwmb0AHJ9k03dSOMxgd99qZsOAxWb2prtvNLPrgPvMLA9YAFTFdnD3p4GnzWwS8H3gkgbqdwNwA8DgwYNTqJKIiKQisqBx96Qf8ABmtt3MBrj7NjMbAHzUwDG2hvebzOxFYByw0d1fAyaGx5oCjEqy71IzG25mfdx9Z5LtvwB+AVBUVKQuNhGRiKSr62we8RP4M4C/1C9gZr3CFgtm1gc4F1gTLvcL7/OAW4GHwuURZmbh4/FAJ6A00r9EREQaFVmLpglzgCfN7MvAFuAzAOF5l39191nAaODnZlZDEIhz3H1NuP8tZnZFuP5Bd18crr8G+BczqwQOAdcmXBwgIiJpYPocDrrOiouL010NEZEOxcxWuHtRU+U0MoCIiERKQSMiIpFS0IiISKQUNCIiEikFjYiIREpBIyIikVLQiIhIpBQ0IiISKQWNiIhEKl1D0IgcM2pqnKoap7rGqaqpoao6teUsM7KzjCwj4XFwH3ucZSRdn51lZJthWZBdb98sg3BoQJFIKWikXbnHPkDD++rgwzS2rv5ydY1TWZ3acnCcFPYJP9iramqOWI7VsbrGqay3HCsfXw73SQiKypoaMnmUp1hAmVlCEHFEWGWZkRWGVVaW1Qmt+uvrbI8dL8n6bCPcnrBfkmMFdUjyHPXCt364ZmUdGcDxsoQBnXx9PMCTr6//t8f/piPDX2GuoEm7xG+7lfU+pJr77Tf+AVqT8GHd/OVkH6xVicdMshzUt+5y/Q/aZB/4VTXp+9TNMsjJyiI7y8jJMrKzjZysrOBxlpGTHayvLVNvOS83J9w3K2H/BpazLf48WUZudlad5ZzshOdNshzbJ8sMd6fGodqdmvB1rH0cbku6PlxX41Dj9bbXxI9X43XX1y8bLCc7Rt1j124P3wcVVfWeO6FMjVO7HKtv4nPE/zanJqxrdRrfO61hSYLWYmF+RACHZY8I1XpBGwu5JOGaHW5Lvv7I8D13RAEXfax/pK+BgqYVXttYygOL1if9tnvkt9/M+7ab7IMt8UM4N/yAjX3oZmdlkRsu5+VmcVzCcv3t8Q/auh/CtWUSl+t8KGeFH7xHLseOF9S17nJ8n4aXs7L0zbKji4eS4x4PoDrhmrg9yfpYcNV4/eCmXojGQ7Z+yHuS9YlBXPvc9dYnhvmRAd34+rp1Cuoba5knhrknfimp9/pU14S9Cgnru3TKVtBkMif4x4p92038YGzut91k336b+rbb0LffVL5Bq0kvHVFWlpGFkZud7ppIKhQ0rXDO8D6cM7xPuqshIpLRdHmziIhESkEjIiKRUtCIiEikFDQiIhIpBY2IiERKQSMiIpFS0IiISKQUNCIiEinzTB7xr52Y2Q7gvRbu3gfY2YbVaSuZWi/I3LqpXqlRvVJzNNbrRHfv21QhBU0rmVmxuxelux71ZWq9IHPrpnqlRvVKzbFcL3WdiYhIpBQ0IiISKQVN6/0i3RVoQKbWCzK3bqpXalSv1Byz9dI5GhERiZRaNCIiEikFTRPMLNvM/mFmzyTZlmdm/21mG8zsdTMbkrDttnD9P83s0nau1zfNbI2ZvWFmi8zsxIRt1Wa2KrzNa+d6zTSzHQnPPyth2wwzWx/eZrRzve5LqNM6M9uTsC2y18vMNpvZm+Gxi5NsNzN7IHwfvWFm4xO2RfZ6NaNenw/r84aZvWpmpzd334jrdYGZ7U3497o9Ydtl4f/FDWY2u53rdUtCnd4K31O9m7NvK+vV08yeMrO1ZvaOmU2ot7393l8eTkuqW/Ib8E3g98AzSbZ9DXgofHwd8N/h45OB1UAeMBTYCGS3Y70uBI4LH/+vWL3C5bI0vl4zgZ8mWd8b2BTe9wof92qvetUr93Xg1+3xegGbgT6NbP8E8CxgwNnA6+3xejWjXufEng+4PFav5uwbcb0uaOB9lx3+HxwGdAr/b57cXvWqV/ZTwOJ2er0eA2aFjzsBPdP1/lKLphFmNhD4JPBwA0WuJPjHBHgKuNjMLFz/B3evcPd3gQ3Ame1VL3df4u4Hw8VlwMC2eu7W1KsRlwIL3X2Xu+8GFgKXpale04En2uq5W+lK4DceWAb0NLMBRPx6NcXdXw2fF9rx/dUKZwIb3H2Tux8G/kDw2qZDu7y/zKw7MAn4FYC7H3b3PfWKtdv7S0HTuPuBfwdqGth+AvA+gLtXAXuBgsT1oZJwXXvVK9GXCb61xOSbWbGZLTOzq9qwTs2t1zVhM/0pMxsUrsuI1yvsYhwKLE5YHeXr5cACM1thZjck2d7Q6xL169VUvRLVf3+lsm8U9ZpgZqvN7FkzOyVclxGvl5kdR/CB/adU922BYcAO4JGwy/hhM+tSr0y7vb9yWrPz0czMrgA+cvcVZnZBQ8WSrPNG1rdXvWJlvwAUAecnrB7s7lvNbBiw2MzedPeN7VSvvwJPuHuFmf0rQWvwIjLk9SLo/nzK3asT1kXyeoXODY/dD1hoZmvdfWli9ZPsE+n7q5n1CipndiFB0JyX6r4R1WslwZAoZWb2CeB/gJFkyOtF0G32irvvasG+qcoBxgNfd/fXzewnwGzgPxLKtNv7Sy2ahp0LTDWzzQRN7YvM7PF6ZUqAQQBmlgP0AHYlrg8NBLa2Y70ws0uA7wBT3b0itt7dt4b3m4AXgXHtVS93L02oyy+Bj4eP0/56ha6jXrdGhK9X4rE/Ap7myO7Vhl6XKF+v5tQLMxtD0BV5pbuXprJvVPVy933uXhY+ng/kmlkfMuD1CjX2/mrr16sEKHH318PlpwiCp36Z9nl/RXES6mi70fBJxhupezHAk+HjU6h7McAm2vhigCbqNY7g5OfIeut7AXnh4z7AetrwpGgz6jUg4fHVwLLwcW/g3bB+vcLHvdurXuG2kwhOzFp7vF5AF6BbwuNXgcvqlfkkdU/WLo/69WpmvQYTnHc8J9V9I67X8bF/P4IP7C3ha5cT/h8cSvxigFPaq17httiX0C7t8XqFx/wbcFL4+E7gP9P1/lLXWYrM7HtAsbvPIzjR9lsz20DwJroOwN3fNrMngTVAFXCj1+2Oibpe/wl0Bf4YXJvAFnefCowGfm5mNQSt2TnuvqYd63WTmU0leE12EVyFhrvvMrPvA38Pd/ue1+1eiLpeEJyk/YOH/9NCUb5e/YGnw3+fHOD37v5c2KWIuz8EzCe4MmgDcBD4UrgtyterOfW6neBc5M/CclUeDMqYdN92rNc04H+ZWRVwCLgu/PesMrP/DTxPcAXar9397XasFwRfrBa4+4Gm9m2jekFwBeXvzKwTQdB+KV3vL40MICIikdI5GhERiZSCRkREIqWgERGRSCloREQkUgoaERGJlIJGREQipaAR6UAsGFa+Twv3nWlmhW1xLJFUKGhEjh0zgcKmCom0NQWNSAuY2RALJpR62ILJrH5nZpeY2SvhZFFnhrdXw9FzXzWzk8J9v2lmvw4fnxbuf1wDz1NgZgvCY/ychAEPzewLZrbcgkmzfm5m2eH6MjP7kZmttGDiu75mNo1ggNXfheU7h4f5eljuTTP7WJSvmRy7FDQiLTcC+AkwBvgY8DmCkYy/BXwbWAtMcvdxBMO2/CDc735ghJldDTwCfNXj8wfVdwfwcniMeQTjjGFmo4FrCUb/HQtUA58P9+kCrHT38cBLwB3u/hRQDHze3ce6+6Gw7M6w3INhvUXanMY6E2m5d939TQAzextY5O5uZm8CQwgGUnzMzEYSDLOeC+DuNWY2E3gD+Lm7v9LIc0wCPh3u9//MLDbh2MUEo1//PRwrqzPwUbitBvjv8PHjwJ8bOX5s24rY84i0NQWNSMtVJDyuSViuIfi/9X1gibtfbWZDCKYZiBkJlNG8cybJBiQ04DF3v62F+8fE6lyNPg8kIuo6E4lOD+CD8PHM2Eoz60HQ5TYJKAjPnzRkKWGXmJldTjBsO8AiYFo4YRZm1tuCGUIh+H8dO+bngJfDx/uBbq34e0RaREEjEp17gf9rZq8QDE8fcx/wM3dfRzBD5ZxYYCRxFzDJzFYCUwjmWCGcruC7BNMAv0Ewr/uAcJ8DwClmtoJgBtPvhesfBR6qdzGASOQ0TYDIUcbMyty9a7rrIRKjFo2IiERKLRqRDGBmXwL+T73Vr7j7jemoj0hbUtCIiEik1HUmIiKRUtCIiEikFDQiIhIpBY2IiERKQSMiIpH6/8HN1eLwAS9NAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb83c4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_2.best_score_, gsearch2_2.best_params_))\n",
    "test_means = gsearch2_2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_2.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_2.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(min_child_weight), len(max_depth))\n",
    "train_scores = np.array(train_means).reshape(len(min_child_weight), len(max_depth))\n",
    "\n",
    "for i, value in enumerate(min_child_weight):\n",
    "    pyplot.plot(max_depth, test_scores[i], label= 'test_min_child_weight:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( '- Log Loss' )\n",
    "pyplot.savefig( 'max_depth_vs_min_child_weght2.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3.重新调整弱学习器数目\n",
    "\n",
    "由于一开始设置的max_depth、min_child_weight与微调后的最优取值相同，为'max_depth': 5, 'min_child_weight': 1，故无需重新设置弱学习器数目，固定为219即可"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4.行列采样参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.6, 0.7, 0.8, 0.9, 1.0],\n",
       " 'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [i/10.0 for i in range(3,10)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,11)]\n",
    "param_test4_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test4_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59029, std: 0.00364, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58787, std: 0.00396, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58641, std: 0.00302, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58579, std: 0.00324, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58563, std: 0.00329, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58486, std: 0.00277, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58541, std: 0.00373, params: {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "  mean: -0.58924, std: 0.00394, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58772, std: 0.00396, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58604, std: 0.00401, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58565, std: 0.00401, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58494, std: 0.00383, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58476, std: 0.00407, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58506, std: 0.00373, params: {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  mean: -0.58876, std: 0.00418, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58774, std: 0.00470, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58703, std: 0.00410, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58578, std: 0.00364, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58520, std: 0.00359, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58451, std: 0.00336, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58505, std: 0.00347, params: {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  mean: -0.58791, std: 0.00448, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58733, std: 0.00396, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58555, std: 0.00347, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58491, std: 0.00399, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58403, std: 0.00376, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58395, std: 0.00385, params: {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       "  mean: -0.58465, std: 0.00336, params: {'colsample_bytree': 0.9, 'subsample': 0.9},\n",
       "  mean: -0.58842, std: 0.00275, params: {'colsample_bytree': 1.0, 'subsample': 0.3},\n",
       "  mean: -0.58713, std: 0.00347, params: {'colsample_bytree': 1.0, 'subsample': 0.4},\n",
       "  mean: -0.58559, std: 0.00395, params: {'colsample_bytree': 1.0, 'subsample': 0.5},\n",
       "  mean: -0.58511, std: 0.00352, params: {'colsample_bytree': 1.0, 'subsample': 0.6},\n",
       "  mean: -0.58417, std: 0.00404, params: {'colsample_bytree': 1.0, 'subsample': 0.7},\n",
       "  mean: -0.58468, std: 0.00364, params: {'colsample_bytree': 1.0, 'subsample': 0.8},\n",
       "  mean: -0.58475, std: 0.00290, params: {'colsample_bytree': 1.0, 'subsample': 0.9}],\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       " -0.5839520712851007)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch4_1 = GridSearchCV(xgb4_1, param_grid = param_test4_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch4_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch4_1.grid_scores_, gsearch4_1.best_params_,     gsearch4_1.best_score_\n",
    "#best {'colsample_bytree': 0.9, 'subsample': 0.8}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([236.04227986, 272.33327713, 300.44743838, 298.32139969,\n",
       "        290.10758262, 283.4964798 , 280.13455992, 267.71007981,\n",
       "        315.47123961, 354.86471844, 321.03103948, 315.64283862,\n",
       "        298.38711972, 285.98007989, 273.78167992, 320.82443852,\n",
       "        330.92447677, 335.72951403, 335.52347794, 354.0615787 ,\n",
       "        319.11721931, 277.50851989, 321.07279992, 341.00223994,\n",
       "        347.48807988, 341.82391973, 334.0112    , 334.21815987,\n",
       "        300.91760001, 344.47702012, 372.31773996, 372.87865996,\n",
       "        371.9665801 , 367.65503993, 344.77884011]),\n",
       " 'mean_score_time': array([0.74419994, 0.74260006, 0.80540009, 0.79380007, 0.76779475,\n",
       "        0.77640009, 0.7494    , 0.81219997, 0.82399993, 1.67420001,\n",
       "        0.73599997, 0.70820007, 0.71039991, 0.68320007, 0.7223999 ,\n",
       "        0.83419995, 0.70560002, 0.70200005, 0.88059998, 0.89980006,\n",
       "        0.63979998, 0.67840009, 0.66960006, 0.67159996, 0.65939999,\n",
       "        0.64660001, 0.65180006, 0.66720004, 0.67260008, 0.67499995,\n",
       "        0.65999999, 0.66500001, 0.65679998, 0.65539994, 0.6132    ]),\n",
       " 'mean_test_score': array([-0.59028544, -0.58786548, -0.58640824, -0.58578757, -0.58562529,\n",
       "        -0.58486402, -0.58541101, -0.58923689, -0.58772465, -0.58603844,\n",
       "        -0.58564853, -0.58494492, -0.58475697, -0.58506273, -0.58876208,\n",
       "        -0.58773903, -0.58702862, -0.58578344, -0.58519764, -0.58451047,\n",
       "        -0.58505168, -0.58790684, -0.58732621, -0.58555425, -0.58490577,\n",
       "        -0.58402594, -0.58395207, -0.58465478, -0.58841607, -0.58713445,\n",
       "        -0.58559293, -0.58510972, -0.58417177, -0.58467999, -0.58475299]),\n",
       " 'mean_train_score': array([-0.51860756, -0.51595991, -0.51413258, -0.5127328 , -0.51225311,\n",
       "        -0.51324682, -0.51463726, -0.51545299, -0.51241582, -0.51122971,\n",
       "        -0.51070619, -0.51095076, -0.51074548, -0.51241982, -0.51338803,\n",
       "        -0.5105549 , -0.5086263 , -0.50800658, -0.50797585, -0.5085567 ,\n",
       "        -0.51066364, -0.51128955, -0.50842466, -0.50710933, -0.50570329,\n",
       "        -0.50592596, -0.50705047, -0.5089418 , -0.50962581, -0.50683053,\n",
       "        -0.50478732, -0.50429056, -0.50484368, -0.50554307, -0.50786777]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.7, 0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9,\n",
       "                    0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 1.0, 1.0, 1.0, 1.0, 1.0,\n",
       "                    1.0, 1.0],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8, 0.9, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.3,\n",
       "                    0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.9},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 1.0, 'subsample': 0.9}],\n",
       " 'rank_test_score': array([35, 30, 24, 22, 19,  9, 16, 34, 28, 23, 20, 11,  8, 13, 33, 29, 25,\n",
       "        21, 15,  4, 12, 31, 27, 17, 10,  2,  1,  5, 32, 26, 18, 14,  3,  6,\n",
       "         7]),\n",
       " 'split0_test_score': array([-0.58493219, -0.58159172, -0.58175637, -0.58031704, -0.57952484,\n",
       "        -0.5803065 , -0.57920484, -0.58284569, -0.58109374, -0.57950275,\n",
       "        -0.57879417, -0.5785109 , -0.57750213, -0.57922142, -0.58164061,\n",
       "        -0.57982495, -0.58012227, -0.57985214, -0.57991476, -0.57861663,\n",
       "        -0.57907289, -0.58010578, -0.58066431, -0.57952063, -0.5782639 ,\n",
       "        -0.57849679, -0.57837307, -0.57899287, -0.58459449, -0.5810362 ,\n",
       "        -0.57897957, -0.57900639, -0.57734903, -0.57878601, -0.57994714]),\n",
       " 'split0_train_score': array([-0.51952483, -0.51725137, -0.51551646, -0.51411581, -0.51281827,\n",
       "        -0.51477381, -0.51497044, -0.51576375, -0.51468624, -0.51239153,\n",
       "        -0.51151681, -0.51173767, -0.51109907, -0.5131065 , -0.5146005 ,\n",
       "        -0.51223679, -0.50966866, -0.50998631, -0.50994015, -0.50974352,\n",
       "        -0.51195542, -0.51279358, -0.50926385, -0.50795652, -0.50531478,\n",
       "        -0.50605524, -0.50734854, -0.50991433, -0.51192113, -0.50823336,\n",
       "        -0.5057979 , -0.50611321, -0.50699538, -0.50679772, -0.5093858 ]),\n",
       " 'split1_test_score': array([-0.58842689, -0.58602745, -0.58430818, -0.58468852, -0.58496489,\n",
       "        -0.58342788, -0.5846533 , -0.58693534, -0.58553819, -0.58520648,\n",
       "        -0.58444684, -0.58327274, -0.58370767, -0.58297957, -0.58687398,\n",
       "        -0.58619334, -0.58527472, -0.58401085, -0.58291226, -0.58346904,\n",
       "        -0.58395987, -0.58611632, -0.58506629, -0.58415516, -0.58425996,\n",
       "        -0.58197104, -0.58200957, -0.58354873, -0.58782345, -0.58699806,\n",
       "        -0.5841971 , -0.58430937, -0.58315548, -0.58282354, -0.58421296]),\n",
       " 'split1_train_score': array([-0.51885943, -0.51609653, -0.51502553, -0.51400508, -0.51341227,\n",
       "        -0.51221403, -0.51384518, -0.51553438, -0.51349961, -0.51257776,\n",
       "        -0.50931327, -0.51040696, -0.51025669, -0.51296406, -0.51360895,\n",
       "        -0.51109942, -0.50801574, -0.50704255, -0.50808517, -0.50906541,\n",
       "        -0.51163549, -0.51214527, -0.50860366, -0.50949081, -0.5081499 ,\n",
       "        -0.50460876, -0.50779133, -0.50862324, -0.50959073, -0.50828321,\n",
       "        -0.50489101, -0.50447343, -0.50376247, -0.50490494, -0.50810089]),\n",
       " 'split2_test_score': array([-0.58972463, -0.58761515, -0.58694202, -0.58593316, -0.58704531,\n",
       "        -0.58539462, -0.58476077, -0.59045116, -0.58940444, -0.5850678 ,\n",
       "        -0.58611405, -0.5855031 , -0.58549651, -0.58516514, -0.58986775,\n",
       "        -0.58811092, -0.58787438, -0.5861157 , -0.58485699, -0.58516549,\n",
       "        -0.58532304, -0.58918367, -0.58910991, -0.58656243, -0.58397559,\n",
       "        -0.58327648, -0.5827669 , -0.58465872, -0.58658315, -0.58668463,\n",
       "        -0.5855645 , -0.5851169 , -0.58383868, -0.58502766, -0.58407428]),\n",
       " 'split2_train_score': array([-0.51848246, -0.51488517, -0.5127906 , -0.51171532, -0.51150414,\n",
       "        -0.51391778, -0.51428591, -0.51615486, -0.51283207, -0.51110938,\n",
       "        -0.51104029, -0.51063472, -0.51130933, -0.51280365, -0.51342309,\n",
       "        -0.50983473, -0.50858358, -0.50787318, -0.50766319, -0.50769202,\n",
       "        -0.51082655, -0.50992437, -0.50846672, -0.50546192, -0.50598083,\n",
       "        -0.50592277, -0.50628879, -0.50868918, -0.50876469, -0.50738179,\n",
       "        -0.50394547, -0.50333325, -0.504408  , -0.50539828, -0.50690614]),\n",
       " 'split3_test_score': array([-0.59280803, -0.59267286, -0.58937581, -0.58957368, -0.58853177,\n",
       "        -0.58791873, -0.58998075, -0.59193987, -0.59045083, -0.58946206,\n",
       "        -0.58820821, -0.5883395 , -0.58830653, -0.58952091, -0.5922439 ,\n",
       "        -0.5908599 , -0.58990128, -0.5889083 , -0.58929492, -0.58717626,\n",
       "        -0.58847708, -0.59217183, -0.59133481, -0.588648  , -0.58943181,\n",
       "        -0.58870955, -0.58814662, -0.58772943, -0.59143486, -0.59003905,\n",
       "        -0.58953187, -0.58891808, -0.58885561, -0.58892356, -0.58763522]),\n",
       " 'split3_train_score': array([-0.51778316, -0.51674008, -0.51347647, -0.51125816, -0.51262023,\n",
       "        -0.51394515, -0.51563729, -0.51572428, -0.51122584, -0.51029592,\n",
       "        -0.51103841, -0.51090438, -0.51087106, -0.51361774, -0.51396996,\n",
       "        -0.5103555 , -0.50874225, -0.50823336, -0.50784777, -0.50809821,\n",
       "        -0.51007458, -0.51111056, -0.50746378, -0.50600321, -0.50458733,\n",
       "        -0.50730837, -0.50689483, -0.50917472, -0.51004472, -0.50624576,\n",
       "        -0.50582102, -0.50421725, -0.50552987, -0.50577512, -0.50692099]),\n",
       " 'split4_test_score': array([-0.59553708, -0.5914213 , -0.58965983, -0.58842623, -0.58806038,\n",
       "        -0.58727311, -0.58845634, -0.59401385, -0.59213737, -0.59095461,\n",
       "        -0.59068091, -0.58909964, -0.58877323, -0.58842763, -0.59318552,\n",
       "        -0.59370786, -0.59197197, -0.59003149, -0.58901043, -0.58812602,\n",
       "        -0.58842656, -0.59195782, -0.59045666, -0.58888608, -0.5885987 ,\n",
       "        -0.58767694, -0.58846557, -0.58834527, -0.59164538, -0.59091548,\n",
       "        -0.58969287, -0.58819882, -0.58766109, -0.58784015, -0.58789633]),\n",
       " 'split4_train_score': array([-0.51838792, -0.51482642, -0.51385381, -0.51256961, -0.51091063,\n",
       "        -0.51138335, -0.51444749, -0.51408771, -0.50983536, -0.50977396,\n",
       "        -0.51062217, -0.51107005, -0.51019125, -0.50960713, -0.51133766,\n",
       "        -0.50924805, -0.50812125, -0.50689748, -0.50634295, -0.50818434,\n",
       "        -0.50882617, -0.51047398, -0.5083253 , -0.5066342 , -0.50448361,\n",
       "        -0.50573466, -0.50692887, -0.50830755, -0.50780781, -0.50400852,\n",
       "        -0.50348121, -0.50331563, -0.50352266, -0.50483928, -0.50802504]),\n",
       " 'std_fit_time': array([12.82982446, 10.48653319, 13.88612847, 16.6305925 , 11.91634954,\n",
       "        14.08757016, 13.23641327, 10.40568514,  9.70300074, 26.21617562,\n",
       "        15.55046561, 14.0431284 , 13.51502951, 12.60685666,  9.89574842,\n",
       "        19.12831104, 10.88097153, 11.14260585, 10.31069313, 23.33901139,\n",
       "         8.51309335,  3.20141993,  7.29348668,  2.11502615,  6.95855556,\n",
       "         4.91212671,  3.81650901,  4.23361937,  4.47864556,  5.25221071,\n",
       "         3.92756773,  5.4687218 ,  6.74716535, 12.230173  , 25.38459758]),\n",
       " 'std_score_time': array([0.042967  , 0.04765554, 0.06221763, 0.05276144, 0.05599416,\n",
       "        0.03113585, 0.03569636, 0.04301349, 0.03876599, 1.12278397,\n",
       "        0.01827564, 0.05488318, 0.05023377, 0.03063603, 0.0315443 ,\n",
       "        0.10079569, 0.03947963, 0.01792207, 0.3735134 , 0.32883454,\n",
       "        0.00462169, 0.0220418 , 0.00991158, 0.02111494, 0.01786166,\n",
       "        0.0114124 , 0.01257614, 0.03020848, 0.00932955, 0.01757263,\n",
       "        0.00952888, 0.01461505, 0.01550997, 0.0079398 , 0.06211403]),\n",
       " 'std_test_score': array([0.00364321, 0.00396375, 0.0030237 , 0.00323895, 0.00328776,\n",
       "        0.00276588, 0.00373286, 0.00394315, 0.00396167, 0.00400592,\n",
       "        0.0040129 , 0.00382854, 0.00407457, 0.00373282, 0.00417593,\n",
       "        0.00470155, 0.00410306, 0.00363984, 0.00359367, 0.00335944,\n",
       "        0.003468  , 0.00447872, 0.00396334, 0.00346558, 0.00398799,\n",
       "        0.00375824, 0.00385402, 0.00335717, 0.00275158, 0.00346819,\n",
       "        0.00394994, 0.00351961, 0.00404496, 0.00364148, 0.00289951]),\n",
       " 'std_train_score': array([0.00057417, 0.00097315, 0.00100216, 0.00116343, 0.00091225,\n",
       "        0.0012497 , 0.0006161 , 0.00071181, 0.00170748, 0.0011111 ,\n",
       "        0.00075185, 0.00045414, 0.00044829, 0.00143257, 0.00110116,\n",
       "        0.00103841, 0.00058816, 0.00110888, 0.00115334, 0.00074349,\n",
       "        0.0011281 , 0.00105406, 0.00057814, 0.00145275, 0.00133767,\n",
       "        0.00086016, 0.00050138, 0.00055994, 0.00137782, 0.00159297,\n",
       "        0.00095028, 0.00102253, 0.00128098, 0.00071435, 0.00091702])}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch4_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.583952 using {'colsample_bytree': 0.9, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd8zdf/wPHXyRIRgiQIQWyJLCRmjVQT1F5RO1VK+1O0pbSq6NRSalR9UaO2UHvvVZssiSZBSIzYEYnIuOf3x03SRHZyQ3Cej8d9PHLv/XzO53w+rbxzzvl83m8hpURRFEVR8kvvZXdAURRFebWpQKIoiqIUiAokiqIoSoGoQKIoiqIUiAokiqIoSoGoQKIoiqIUiAokiqIoSoGoQKIoiqIUiAokiqIoSoEYvOwOvAgWFhbSxsbmZXdDURTllXLu3Ll7UkrLnLZ7IwKJjY0NZ8+efdndUBRFeaUIIa7lZjs1taUoiqIUiAokiqIoSoGoQKIoiqIUyBuxRqIoRUFCQgIRERHExcW97K4oSjrGxsZYW1tjaGiYr/1VIFGUFyQiIoKSJUtiY2ODEOJld0dRAJBScv/+fSIiIqhWrVq+2lBTW4rygsTFxWFubq6CiFKkCCEwNzcv0EhZBRJFeYFUEFGKooL+f6kCSRaklKw7E86ei7dfdlcURVGKNLVGkoUkjWT5yWvcfPSUhlXLYG5a7GV3SVEUpUhSI5IsGOjrMb2XE9FxiXyz5eLL7o6iFNijR4+YN29evvb97bffiI2N1Vlfli5dyogRI3TWXm4dOnSIjh075nm/vPY3LCyMVatW5fk4ufXgwQPc3d2pVasW7u7uPHz4MNPtrl+/joeHB7a2ttjZ2REWFlYo/VGBJBt1KpRktHsttvvdYpvfzZfdHUUpkKIUSF532QWSxMTEArc/depU2rRpQ0hICG3atGHq1KmZbjdw4EDGjh1LUFAQp0+fply5cgU+dmbU1FYOPmxRnd0XI5m4KYDG1cyxLKmmuJSCm7L1IoE3H+u0TbuKpZjUqV6W348fP57Lly/j7OyMu7s75cqVY926dTx79oxu3boxZcoUYmJi8PT0JCIigqSkJCZOnEhkZCQ3b97Ezc0NCwsLDh48mGn7u3bt4quvviIpKQkLCwv279/PgwcPGDx4MFeuXMHExIQFCxbg6OiYbj9vb2+mTJmCvr4+ZmZmHDlyhLCwMAYMGEBMTAwAc+fOpVmzZhw6dIhJkyZRvnx5fHx86N69Ow4ODsyaNYunT5+yadMmatSogZeXF8bGxly8eJHIyEhmzJiRYSQSExPDJ598gr+/P4mJiUyePJkuXbpkef3Cw8Np164dV69epW/fvkyaNImJEydiYWHBqFGjAJgwYQLly5dn1apVBAUF4ezszKBBgyhTpgzbt28nLi6OmJgYDhw4wLRp0zJcf4AVK1Ywe/Zs4uPjady4MfPmzUNfXz9dXzZv3syhQ4cAGDRoEK1bt+bnn39Ot01gYCCJiYm4u7sDYGpqmuW5FZQKJDkw0Nfj116OvDv7GF9v8md+/4bqzhvllTR16lQCAgLw8fFhz549rF+/ntOnTyOlpHPnzhw5coS7d+9SsWJFtm/fDkBUVBRmZmbMmDGDgwcPYmFhkWnbd+/eZejQoRw5coRq1arx4MEDACZNmkT9+vXZtGkTBw4cYODAgfj4+KTb99tvv2X37t1UqlSJR48eAVCuXDn27t2LsbExISEh9OnTJzXxqq+vL0FBQZQtW5bq1aszZMgQTp8+zaxZs5gzZw6//fYboB0VHD58mMuXL+Pm5kZoaGi64/7www+8/fbbLF68mEePHtGoUSPeeecdSpQokek5nj59moCAAExMTHB1daVDhw588MEHdO/enVGjRqHRaFizZg2nT5/G0dGR6dOns23bNkA7NXbixAn8/PwoW7Yse/bsISQkJMP1t7S0ZO3atRw/fhxDQ0M+/vhjVq5cycCBAxkyZAjDhw/HxcWFyMhIrKysALCysuLOnTsZ+hscHEzp0qXp3r07V69e5Z133mHq1KkZgpIuFGogEUK0A2YB+sAiKeXU5773AqYBN5I/miulXJT83c9Ah+TPv5NSrk3+vBqwBigLnAcGSCnjC/M8apYryefutflp5yW2+N6ki3Olwjyc8gbIbuTwIuzZs4c9e/ZQv359AJ48eUJISAgtWrRgzJgxjBs3jo4dO9KiRYtctXfy5ElatmyZ+kBb2bJlATh27BgbNmwA4O233+b+/ftERUWl27d58+Z4eXnh6elJ9+7dAW0WgBEjRuDj44O+vj7BwcGp27u6uqb+Eq1RowYeHh4AODg4pBsteXp6oqenR61atahevTqXLl3KcA22bNnC9OnTAe1zPtevX8fW1jbTc3R3d8fc3ByA7t27c+zYMUaPHo25uTkXLlwgMjKS+vXrp26T2f4p1yWr6+/n58e5c+dwdXUF4OnTp6nTUYsWLcq03awkJiZy9OhRLly4QJUqVejduzdLly7lgw8+yFM7uVFogUQIoQ/8DrgDEcAZIcQWKWXgc5uulVKOeG7fDkADwBkoBhwWQuyUUj4GfgZmSinXCCHmAx8AfxTWeaQY0qI6uy7e5pvNF2la3ZxypYwL+5CKUmiklHz55ZcMGzYsw3fnzp1jx44dfPnll3h4ePDNN9/kqr3MRupSygyfPb/d/PnzOXXqFNu3b8fZ2RkfHx/mzJlD+fLl8fX1RaPRYGz837+3YsX+m17W09NLfa+np5du/eH54zz/XkrJhg0bqFOnTo7nl117Q4YMYenSpdy+fZvBgwdnuX/akU5W13/OnDkMGjSIn376Kdu+lC9fnlu3bmFlZcWtW7cyXfuwtramfv36VK9eHYCuXbty8uTJQgkkhbnY3ggIlVJeSR4xrAGynoBMzw44LKVMlFLGAL5AO6H9L/c2sD55u2VAVx33O1P6eoLpvZyIS0jiq43+mf4DUZSirGTJkkRHRwPQtm1bFi9ezJMnTwC4ceMGd+7c4ebNm5iYmNC/f3/GjBnD+fPnM+ybmaZNm3L48GGuXr0KkDq11bJlS1auXAlo75iysLCgVKlS6fa9fPkyjRs35ttvv8XCwoLw8HCioqKwsrJCT0+P5cuXk5SUlOfz9fb2RqPRcPnyZa5cuZIhYLRt25Y5c+ak/lu+cOFCtu3t3buXBw8epK7FNG/eHIBu3bqxa9cuzpw5Q9u2bYGcr1dW179NmzasX78+darqwYMHXLuWsSRI586dWbZsGQDLli3LdG3H1dWVhw8fcvfuXQAOHDiAnZ1dtueYX4U5tVUJCE/zPgJonMl2PYQQLYFg4FMpZTjawDFJCDEDMAHcgEDAHHgkpUxM0+YLm2eqYWnK2LZ1+H57EBsv3KB7A+sXdWhFKTBzc3OaN2+Ovb097du3p2/fvjRt2hTQLsSuWLGC0NBQxo4di56eHoaGhvzxh3aw/+GHH9K+fXusrKwyXWy3tLRkwYIFdO/eHY1Gk7rGMXnyZN5//30cHR0xMTFJ/eWX1tixYwkJCUFKSZs2bXBycuLjjz+mR48eeHt74+bmluW6RXbq1KlDq1atiIyMZP78+elGNQATJ05k9OjRODo6IqXExsYmdU0jM2+99RYDBgwgNDSUvn374uLiAoCRkRFubm6ULl06df3B0dERAwMDnJyc8PLyokyZMuna8vDwICgoKMP1t7Oz4/vvv8fDwwONRoOhoSG///47VatWTbdGMn78eDw9Pfnzzz+pUqUK3t7eAJw9e5b58+ezaNEi9PX1mT59Om3atEFKScOGDRk6dGier2NuiML6y1oI0QtoK6Uckvx+ANBISvlJmm3MgSdSymdCiOGAp5Ty7eTvJgC9gLvAHeA0sAo4IaWsmbxNZWCHlNIhk+N/CHwIUKVKlYaZRfX8SNJI3ltwgn9vR7Pn01ZUMFNTXEruBAUFZTn/ruiWl5cXHTt2pGfPnoV+LI1GQ4MGDfD29qZWrVqFfrzCktn/n0KIc1JKl5z2LcyprQigcpr31kC6hzGklPellM+S3y4EGqb57gcppbOU0h0QQAhwDygthDDIqs00+y+QUrpIKV0sLXMsOZxr+nqCaT2diE/S8OXffmqKS1HeYIGBgdSsWZM2bdq80kGkoApzausMUCv5LqsbwHtA37QbCCGspJS3kt92BoKSP9cHSksp7wshHAFHYI+UUgohDgI90a65DAI2F+I5ZMrGogTj29Vl8tZAvM9F4OlSOeedFOU10bhxY549e5bus+XLl+PgkGFi4KVZunRpvvbbvXs348aNS/dZtWrV2LhxY6bb29nZceXKlXwd63VSaIFESpkohBgB7EZ7++9iKeVFIcS3wFkp5RZgpBCiM5AIPAC8knc3BI4m3xXxGOifZl1kHLBGCPE9cAH4s7DOITsDm9qwM+A2320N5K2aFlQsXfxldENRXrhTp0697C4UmrZt26YumCu5V2hrJEWJi4uLTHmYSZeu34+l3awjNKxahr8GN1IPKirZUmskSlFWVNdIXntVzE348l1bjobcY82Z8Jx3UBRFeQ2pQFJA/RpVoXlNc77fFkjEQ5XUTlGUN48KJAWkpyf4uYc2Cd24DeouLkVR3jwqkOiAdRkTvu5ox/HQ+6w8df1ld0dRMlWU0sireiQFk5t6JAcPHsTZ2Tn1ZWxszKZNmwqlPyqQ6Mh7rpVpUcuCH3cEEf5ATXEpRU9RCiSvu6JQj8TNzQ0fHx98fHw4cOAAJiYmqQkudU2lkdcRIbRTXG1nHmHsel9WDWmCnp66i0vJws7xcNtft21WcID2mRc4AlWP5E2rR5LW+vXrad++PSYmJlluUxAqkOhQxdLFmdjRji82+LH85DUGNbN52V1SlFSqHsmbVY8krTVr1vDZZ59lu01BqECiY71crNkRcIupOy/RqrYlNhZ5TzanvAGyGTm8CKoeyetfjyTFrVu38Pf3L9QHLVUg0TEhBFO7O+I+8zBj1/uy9sOmaopLKXJUPZLXvx5JinXr1tGtWzcMDQ2zbbMg1GJ7IahgZszkTvU4E/aQJf+EvezuKAqg6pG8afVIUqxevZo+ffpke24FpUYkhaR7g0rsDLjFL7su4VbHkuqWpi+7S8obTtUjebPqkYB2nSg8PJxWrVrl+frlhcq1VYjuPI7DfeYRaliWwHt4M/TVFNcbTeXaenFUPZK8U7m2iqhypYz5tks9zl9/xJ/HVKppRXndqHokWmpqq5B1dqrIDv9bTN8TzNt1y1GzXMmX3SVFKRBVj+Q/qh6JlpraegHuRj/DY+ZhqpiXYMPwphjoq4Hgm0hNbSlFmZraKuIsSxbju672+IY/YsFR9deLoiivFxVIXpCOjhXp4GDFb3tD+Pd21rcFKoqivGpUIMmOJu/3rmfn2y71KGlswBhvXxKSNDptW1EU5WVRgSQrmiRYPxgOfA86WkcyNy3G913t8b8RxfxDl3XSpqLkVlHK/qvSyBdMbtLIA3zxxRfUq1cPW1tbRo4cWWj1klQgyY5xKTgyDXaMAY1uRhDtHazo5FSR2QdCCLr1WCdtKkpuFKVA8rorCmnk//nnH44fP46fnx8BAQGcOXOGw4cPF/jYmVGBJCt6+tBpNjQfBWcWwcYPISlBJ01/27keZsWN+HydmuJSXpy0aeTHjh3LtGnTcHV1xdHRkUmTJgHa1OodOnTAyckJe3t71q5dy+zZs1PTyLu5uWXZ/q5du2jQoAFOTk60adMG0P7l3LVrVxwdHWnSpAl+fn4Z9vP29sbe3h4nJydatmwJaH8Rt2jRggYNGtCgQQP++ecfQDuiaNWqFZ6entSuXZvx48ezcuVKGjVqhIODA5cva0f6Xl5eDB8+nBYtWlC7du1Mn1iPiYlh8ODBuLq6Ur9+fTZv3pzt9UtJI1+nTp3UlO8TJ05k1qxZqdtMmDCB2bNnM378eI4ePYqzszMzZ85k6dKl9OrVi06dOqUmmczs+oM2jXyjRo1wdnZm2LBhmaaH2bx5M4MGDQK0aeQzK1glhCAuLo74+HiePXtGQkIC5cuXz/Yc80s9R5IdIcD9WyheBvZNhrjH0GspGBUsp3+ZEkb82M2eD5ef4/eDoYx+p7ZOuqu8On4+/TOXHlzKecM8qFu2LuMajcvye5VG/s1KI9+0aVPc3NywsrJCSsmIESMK7fZzFUhy461Pwbg0bPsUVvSAvmvA2KxATXrUq0C3+pWYeyCUd2zLY1+pYO0pSl6oNPKvfxr50NBQgoKCiIiISD3+kSNHUkd9ulSogUQI0Q6YBegDi6SUU5/73guYBtxI/miulHJR8ne/AB3QTr/tBUZJKaUQojcwIbnN7VLKLwrzHFK5vK9dM/l7GCztCP3/BlPLAjU5qZMdx0PvMcbbly0j3sLIQM00vimyGzm8CCqN/OufRn7jxo00adIEU1Ntwtj27dunBnxdK7TfXEIIfeB3oD1gB/QRQthlsulaKaVz8isliDQDmgOOgD3gCrQSQpijDTxtpJT1gPJCiDaFdQ4Z2PeAPmvgXggsaQePwgvUXGkTI37q7sCl29HMORCio04qSuZUGvk3K418lSpVOHz4MImJiSQkJHD48OFCm9oqzD+BGwGhUsorUsp4YA2QddL89CRgDBgBxQBDIBKoDgRLKe8mb7cP6KHTXuek1jswcBM8uQuL22mDSgG0sS1Pz4bWzDt0Gb+IRzrqpKJklDaN/N69e1PTyDs4ONCzZ0+io6Px9/dPXej94Ycf+Prrr4H/0shntdieNo28k5MTvXv3BmDy5MmcPXsWR0dHxo8fn2UaeQcHB+zt7WnZsmVqGvlly5bRpEkTgoODC5RGvn379lmmkU9ISMDR0RF7e3smTpyYbXspaeSdnZ3p0aNHhjTynp6emaaRnzlzZoa2PDw8Mr3+adPIOzo64u7uzq1btwDtyCdlnWj8+PHs3buXWrVqsXfvXsaPHw9o08gPGTIEgJ49e1KjRg0cHBxwcnLCycmJTp065fk65oqUslBeQE+001kp7wegnbpKu40XcAvwA9YDldN8Nx14BEQBPyR/VgaIAGzQTsttALZmcfwPgbPA2SpVqkidu+Un5S81pPy5mpQ3LhSoqUex8bLxD/vkO78eknEJiTrqoFLUBAYGvuwuvDEGDRokvb29X8ixkpKSpJOTkwwODn4hxyssmf3/CZyVufh9X5gjksyKbzw/YboVsJFSOqIdXSwDEELUBGwBa6AS8LYQoqWU8iHwEbAWOAqEAZnelC2lXCCldJFSulhaFmwtI1MVHGDwbjAsoV0zCTuW76bMihsytYcDIXee8Ns+NcWlKK8KlUZeqzAX2yOAymneWwM3024gpbyf5u1C4Ofkn7sBJ6WUTwCEEDuBJsARKeVWtAEIIcSHgG7zmOSFeQ34YDf81VV7N1evZVCnXb6aal2nHO+5VuZ/hy/jYVee+lXK5LyTorwEKo38f1Qaea1CSyMvhDAAgoE2aO/KOgP0lVJeTLONlZTyVvLP3YBxUsomyXdmDQXaoR3Z7AJ+k1JuFUKUk1LeEUKUAQ4CnlLKYLJR6GnkY+7Dyp5wyxe6zQdHz3w1Ex2XQNuZRyhupM/2kS0wNtTXcUeVl0mlkVeKsiKZRl5KmQiMAHYDQcA6KeVFIcS3QojOyZuNFEJcFEL4AiPRrpmAdr3kMuAP+AK+ySMRgFlCiEDgODA1pyDyQpQwh0FboGoz+HsonF6Yr2ZKGhvyS08nLt+NYcbel39aiqIouVGoz5FIKXcAO5777Js0P38JfJnJfklAxhvctd/10XE3daNYSei3XpvocccYePoIWo7RPh2fB2/VsqBf4yosPHqFtvXK07Bq2ULqsKIoim6oJ+CyEfUsKm/ZMg2NwfMvcOoDB7+H3V/lK9njl+/aUql0ccZ4+/E0/uUtASmKouSGCiRZSNIk8fG+jxm6dyjXH1/P/Y76BtBlHjQeDifnwZYRkJS3bJ+mxQz4pacjV+/FMG33v3nsuaIoyoulAkkWhBB0rtGZi/cu0n1Ldxb6LSQht9l/9fSg3VRo/RX4rATvQZAQl6fjN6thwcCmVVnyz1VOX32QjzNQlPSKUhp5VY+kYHJbj2TcuHHY29unZnIuLCqQZEFP6NG7bm82d91MS+uWzL4wG89tnvjc8cl5Z9CujbQeB+1/gUvbYFUveJa3Ervj2tWlchkTxq73JTa+4DUMlDdbUQokr7uiUI9k+/btnD9/Hh8fH06dOsW0adN4/LhwaiCpQJKDciblmNF6BnPensOThCcM3DmQ709+T3R8LoNC42HQbQGEHYdlnSE296OLEsUMmNbTkWv3Y/lll5riUgpG1SNJ73WvRxIYGEirVq0wMDCgRIkSODk5sWvXrmzPMb9UGvlcal25Na4VXJl7YS6rLq3iwPUDjG80Hveq7plmPU3Hqbc2c/C6QbCkPQzYCKUq5uq4jaub835zG5YcD6NtvQo0rZF5imrl1XL7xx95FqTbeiTFbOtS4auvsvxe1SN5s+qRODk5MWXKFD777DNiY2M5ePAgdnaZ5c0tODUiyYMShiUY12gcq95dhXlxcz4//DmfHPiEW09u5bxznfbQfwNE3YDFbeF+7mu2f9G2Ljbm2imumGdqikspuLT1MBo0aMClS5cICQnBwcGBffv2MW7cOI4ePYqZWe7q5GRXj2TAgAFAzvVIFi5cmPrXd0JCAkOHDsXBwYFevXoRGBiYun1KPZJixYplqEcSFhaWul1u6pFMnToVZ2dnWrdunVqPJCsp9UiKFy+eWo/ExsYmtR5JyvXMaz2StNd///79qfVInJ2d2b9/f+qT84sWLUpNFJkbHh4evPvuuzRr1ow+ffrQtGlTDAwKZ+ygRiT5UM+iHqs7rGZl0Ep+9/mdLpu78En9T+hbty/6etk8jV6thfbBxRU9tJmDB2yECvY5Hq+4kT7TeznR638n+GlnEN93LTqpKJT8yW7k8CJIVY/kta9HAtqptgkTJgDQt2/fQssHpkYk+WSgZ8CgeoPY2GUjLuVd+OXML/Td0ZfA+4HZ71ipAQzeBfqGsPRduH4qV8dzsSnLkLeqseLkdY6F3NPBGShvGlWP5M2qR5KUlMT9+9p0hn5+fvj5+aWO3nRNjUgKqJJpJX5v8zu7r+1m6qmp9Nneh/62/fk/5//DxDCL2u6WdbTB5K+u8FcXeG8F1Hwnx2N97lGH/ZfuMG6DH7tGt6CksaGOz0Z5naWtR9K+ffvUehgApqamrFixgtDQUMaOHYuenh6Ghob88ccfwH/1SKysrNKVs02Rth6JRqNJXeOYPHky77//Po6OjpiYmGRZjyQkJAQpJW3atEmtR9KjRw+8vb1xc3MrUD2SyMjILOuRjB49GkdHR6SU2NjYZLoonyKlHkloaCh9+/bNUI+kdOnSmdYj8fLyokyZ9ElYPTw8CAoKynD909Yj0Wg0GBoa8vvvv1O1atV0ayTjx4/H09OTP//8kypVquDt7Q1o65HMnz+fRYsWkZCQkFoquVSpUqxYsaLQprYKLWljUVLoSRuTRT2L4rfzv7E+eD1WJaz4usnXtLTOpqzlk7uwohvcuQTdF4B99xyPcf76Q3r+8Q+9XSvzU3dHHfZeKWwqaeOL4+XlRceOHenZs2ehH0uj0dCgQQO8vb1f6VTyRTJp4+vg/s0nPHua+8Vts2JmTGo6iWXtlmFiYML/7f8/Pj/0OXdj72a+g6kleG0Haxdtjq5zS3M8RoMqZfiwZQ1Wnw7ncHAW7SqK8kKoeiRaakSShaQkDasmnUToCdoPd8C8omme9k9ISmBxwGIW+C2gmH4xRjccTc/aPdETmcTu+FhYNxBC98I7U+Ct0dm2HZeQRKc5x4iOS2T3py0xK66muF4Fr8uI5FWoR5Jfea1H8jopyIhEBZJs3Ax5xK6FASQ8S+LtAXWp5VI+z22ERYXx3cnvOH37NM6WzkxqOomaZWpm3DAxHjYNh4AN0Hw0vDM528zBvuGP6P7HP3SvX4lpvZzy3C/lxXtdAonyelJTW4WkYq3S9P7KFYtKJdiz6CLHN4SiScpbNl8bMxsWeSziu+bfcfXxVXpt68Xs87N5lpT+LzoMjKD7QnAZDMd/g22jQZP1nSpOlUvzUasaeJ+L4MClyPycnqIoik6oQJKDEqWL0fWzBti3qoTP3utsme3D0+j4PLUhhKBrza5s6bqF9jbtWei/kO6bu3Pq1nO3/urpQ4cZ0OJz7XrJ+sHakUoWPmlTk7oVSjJ+gz9RsblMKKkoiqJjKpDkgr6BHq361OHtgbbcvvyYdT+eITIs78nPyhqX5ccWP7LAfQESyZA9Q5hwbAIP49Jk7hQC2nwDHt9D4CZY/R7Ex2TaXjED7YOK92PimbL1YqbbKIqiFDYVSPLAtpkVPb5oiBCCv6efI/D4zXy107RiU/7u/DdDHYay48oOOm/qzJbLW9I/BdzsE+g8F64chOXd4GnmaaLtK5nxf241+fvCDfZcvJ2v/iiKohSECiR5ZFmlJL2+cqFizdIcXH6JgysvkZSQ9yqIxgbGjGwwknWd1lG1VFUmHJvA0L1DufY4zVOsDQZAr6Vw8wIs7QjRma+FjHCria1VKb7aGMDDmLxNuylvjqKURl7VIykYb29v6tWrh56eHtndSLRr1y7q1KlDzZo1M001rysqkORDcVMjOo10pkG7qgQevcnfv54n+kHeClelqFWmFn+1/4uJTSZqi2htfq6Ill0X6LsWHlyFJe3gYcZ0CUYGevzay4lHsfFMVlNcShaKUiB53RV2PRJ7e3v+/vvv1LT7mUlKSuL//u//2LlzJ4GBgaxevTpd8ktdUilS8klPT9C0aw3KVS3J/qVBeP90hrZD7KlUp0zOOz/fltDDs44nrSu3Zurpqcy+MJsdV3cwqekknMs5Q423YeBmWNlTmzl4wCYoVzddG3YVSzGyTS1m7A2mvX0F2tlb6epUlUJwdF0w98Kf6LRNi8qmtPCsneX3aeuRuLu7U65cOdatW8ezZ8/o1q0bU6ZMISYmBk9PTyIiIkhKSmLixIlERkam1iOxsLDINEUKaP/6/eqrr0hKSsLCwoL9+/fz4MEDBg8ezJUrVzAxMWHBggU4OqbPyODt7c2UKVPQ19fIlxBIAAAgAElEQVTHzMyMI0eOEBYWxoABA4iJ0a4Pzp07l2bNmnHo0CEmTZpE+fLl8fHxoXv37jg4ODBr1qzUHFg1atTAy8sLY2NjLl68SGRkJDNmzMgwEomJieGTTz7B39+fxMREJk+enGnOqhQp9UiuXr1K3759mTRpEhMnTsTCwoJRo0YB2iSJ5cuXZ9WqVQQFBeHs7MygQYMoU6YM27dvJy4ujpiYGA4cOMC0adMyXH/Q1iOZPXs28fHxNG7cmHnz5qWmXkmRm9vIT58+Tc2aNalevToA7733Hps3by6UVPJqRFJANeqXo+d4F4xLGLJ5lg8++65nmvE0N1KKaM19ey5PEp4wYOcAvjvxHY/jH0NlV3h/B0iNdmQScS7D/h+1roF9pVJM2BjA/SfPMjmC8iabOnUqNWrUwMfHB3d399R6GD4+Ppw7d44jR46wa9cuKlasiK+vLwEBAbRr146RI0dSsWJFDh48mGUQSalHsmHDBnx9fVNzP6XUI/Hz8+PHH39k4MCBGfZNqUfi6+vLli1bgP/qkZw/f561a9cycuTI1O19fX2ZNWsW/v7+LF++nODgYE6fPs2QIUOYM2dO6nYp9Ui2b9/O8OHDiYtLP2uQUo/kzJkzHDx4kLFjx6YGrsycPn2alStX4uPjg7e3N2fPnuWDDz5IzR+WUo+kX79+TJ06lRYtWuDj48Onn34KwIkTJ1i2bBkHDhxIV48k7fUPCgpKrUfi4+ODvr5+atLLIUOGZDuN9bwbN25QuXLl1PfW1tbcuHEj1/vnRaGOSIQQ7YBZgD6wSEo59bnvvYBpQMrZzZVSLkr+7hegA9pgtxcYJaWUQog+wFeABG4C/aWULzUdblmrEvQc58L+ZUEcXx9KZNhj3PrXxcg4f5e3VeVW2iJaPnNZGbSSA+HaIloeVT0Qg3drEz3+1RneWwXVW6XuZ6ivx6+9nOk45yjfbLnI730b6OoUFR3LbuTwIqSthwHw5MkTQkJCaNGiBWPGjGHcuHF07NgxNelfTrKrR7JhwwYg53oknp6edO+uzTeXkJDAiBEjUn+ZBgcHp26fUo8EyFCPJG2gy009ki1btjB9+nSA1HokWf21n1KPBEitRzJ69OjUeiSRkZH5qkcC/11/Pz+/1HokAE+fPk1NEb9o0aJM281KblL460qhBRIhhD7wO+AORABnhBBbpJTPT9KtlVKOeG7fZkBzIGUMfAxoJYQ4hjYw2Ukp7yUHmxHA5MI6j9wyKm5Au2H2nN99jVObr/DgZgzthzlQunwWGYBzYGJowheuX9Chegem/DOFMYfH0NK6JRMaT6Di4N2wort2qqvnErD9b8hep0JJRr9Tm2m7/6W9/U06OuauEqPyZlH1SF7teiS5YW1tTXh4eOr7iIgIKlYsnN8HhTm11QgIlVJekVLGA2uArCcg05OAMWAEFAMMgUhAJL9KCO1/xVJoRyVFghCChu1s6PSJMzFRz/CeepYwv4INluqZ12NVh1WMcRnDmdtn6Lq5K8vC95A4cDNYOcG6AeCTflFvWMvqOFmbMXFTAHej1RSXoqXqkbw+9Uhyw9XVlZCQEK5evUp8fDxr1qyhc+fO+WorJ4UZSCoB4WneRyR/9rweQgg/IcR6IURlACnlCeAgcCv5tVtKGSSlTAA+AvzRBhA74M9CPId8qWxXFs8vXTGzLM72eX6c2noFqcl/TrOUIlqbumzCtYIr089Op+/BEVzsMBWqtYJNH8HJP/7bXl+P6b2ciIlP4utN/vles1FeL2nrkezduze1HomDgwM9e/YkOjoaf39/GjVqhLOzMz/88ANff/018F89Ejc3t0zbTluPxMnJid69ewMwefJkzp49i6OjI+PHj8+yHomDgwP29va0bNkytR7JsmXLaNKkCcHBwQWqR9K+ffss65EkJCTg6OiIvb09EydOzLa9lHokzs7O9OjRI0M9Ek9Pz0zrkcycOTNDWx4eHple/7T1SBwdHXF3d+fWLW0p77RrJBs3bsTa2poTJ07QoUOH1AB28+ZN3n33XQAMDAyYO3cubdu2xdbWFk9PT+rVq5fn65grUspCeQG90K6LpLwfAMx5bhtzoFjyz8OBA8k/1wS2A6bJrxNAS7Qjk/1ADbQjk7nA11kc/0PgLHC2SpUq8mVIeJYo9y25KOcO2y+3zvGRT5/EF7hNjUYjd17dKVuvbS0dlznKn0/+KGNW95FyUikpD/wgpUaTuu38Q6Gy6rhtctOFiAIfVym4wMDAl92FN8agQYOkt7f3CzlWUlKSdHJyksHBwS/keIUls/8/gbMyF7/vC3NEEgFUTvPemuemoaSU96WUKXMvC4GGyT93A05KKZ9IKZ8AO4EmgHPyfpeTT3Id0Cyzg0spF0gpXaSULpaWlvk6gZjTp0m4mf+ZMwMjfd4eZEurPrUJD3qA909nuBdRsFs+hRC0s2nH5q6b6VGrB8svraKrwV0O27eHwz/Dzi9Ao31AckiL6tSvUppvNl/kzuP8PeeiKErWVD0SrUJLIy+EMACCgTZo78o6A/SVUl5Ms42VlPJW8s/dgHFSyiZCiN7AUKAd2pHHLuA34Fzyy1FKeVcI8R1gIqX8PLu+5CeNvJSS0DZtSLx5CxMXF0p17kSptm3RNzPLUzspbl2OYtcCf+JjE3EbUJfajSrkq53nXbhzgSn/TOFy1GXci1nxZcgZLO16Qtd5oG/I5btPeHfWUVrUsmDhQJdCu2tDydnrkkZe1SN5PRXZeiRCiHfRBgB9YLGU8gchxLdoh0tbhBA/AZ2BROAB8JGU8lLyHV/z0E5nSWCXlPKz5DaHA6OABOAa4CWlvJ9dP/JbjyT++nWitm3j8dZtxF+9ijA0xLR1K0p17IRp61bopbl7JDdiop6xe2EAt0KjcHq7Mk171EBfv+CDwoSkBJZcXML/fP+HkZR8ejeSnhWao+e5DAyLs+joFb7fHsQYj9r8n1tNFUxektclkCivpyIbSIqKgtZsl1ISF3CRx9u2ErV9B0n37qFXsiQl23pg1qkzJq4uCL3cBYSkJA3/rA/F72AEFWuVxmNIPUqY5S0gZeXa42t8d+I7Tt0+hXPcM74xqkqtPhtIMirJZ+t82OxzE69mNkzsaIe+ngomL1pQUBB169ZVgVwpcqSUXLp0SQWS7BQ0kKQlExOJOXmKx1u3EL13H5rYWAwqVMCsYwdKdeqEcS7vSf/31G0OrbhEMRMD2g1zoEL1/E2ZZeiflGy9spVpJ77nSWIs7ycV58NemzAyrchPO4NYePQq7zpUYIanM8aG+jk3qOjM1atXKVmyJObm5iqYKEWGlJL79+8THR2d+kBpChVI0tBlIElL8/Qp0QcO8HjLVp4cPw6JiRSrXZtSnTpi1rEjhlbZ57u6FxHNzvn+PHn4jBa9a1OvRUWd/YJ5GPeQ6Qc+Y8vds1RJEkxs+QNNanZi4ZEr/LAjiMbVyrJgoIuq9/4CJSQkEBERkSFVh6K8bMbGxlhbW2NomP73gc4CiRCiBhAhpXwmhGiN9mnzv6SUj/Lf7RersAJJWokPHvB4504eb93GUx8fAExcXf9bpH/uIawUcTEJ7F18kesXH1C3mRWt3quNgZHuRgonff7ku3O/ct1An6YWTvR3+pD7d6vxxXp/aliasvT9RlQwM865IUVR3ji6DCQ+gAtgA+wGtgB1pJTv6qCfL8SLCCRppS7Sb9lKfFjYf4v0nTph2irjIr1GIzmz7Spnd4RhWaUk7YbZU8q8uM76ExdxhhWb+rO6hCF39AQ2pWxoYt6FVfstMTM25a8PGlGzXEmdHU9RlNeDLgPJeSllAyHEWCBOSjlHCHFBSllfV50tbC86kKTIapG+VLu2lOrYKcMi/VW/e+xbfBE9fT08htSjsm1Z3XXmtj8JSzuwt6QZy61rE/AomBIGpsQ9cEUT1ZwlA9xpWFWHx1MU5ZWny0ByCu0tvBOATlLKq0KIACmlvW66WvheViBJK+0i/eO9+5DpFuk7Y1xHmw32UWQsO//nz8NbMTTpWoP6HlV0tzB74xws64IsWQ7fzr+yImw7e6/tQ6ORaJ7UY6TLYD5s3EYtBCuKAug2kNihTV9yQkq5WghRDegtn0sJX5QVhUCSliY2lugDB3m8dStPjh2DpKR0i/SyjCUHl18i9NwdatS35O1BtvlOSZ/BtRPazMFlbGDQNm6TwGK/Faz91xuNiKWCcS1GuQ6mbdW2GOqrhXhFeZMVyl1bQogyQGUppV9BOveiFbVAklbqIv2WrTz19QUhMHFxoWSnToQVd+LkjghKlzeh/XAHylTIe+K6TF05DKs8waI2DNoKxUtzPzaagWv/4GrCLvSL3cWyuCXv1X2PXrV7UcY471UfFUV59elyRHII7dPnBoAPcBc4nPKk+augKAeStOKvXydq61btk/TJi/RPW/bknH5zNMKAd7zsqF4/f3nDMgjZC6v7QEVnGLARipUkIUnDlxt82fjvQapWO8fdJD+K6RejY/WO9LPtR60yb24uIUV5E+kykFyQUtYXQgxBOxqZJITwk1I6ZrtjEfKqBJIU2kX6AG1Q2bGTmOhEAhyH87hEZRwdDWn2YTP0DXRwi3DQVlg3CKo0gX7rwcgEKSW/7glm7sFQmtkmULOmLzuvbiMuKY4mVk0YYDeAtyq9hZ5QVZoV5XWny0DiD3gAy4AJUsozKpC8ODIxkZgTJ3m4dTtnQ0txs1xjzGMu09whBssu76Yu0ueb/3r4e6i2rkmfNWCofaZk+YkwvtlyEefKpZn5Xi323djC6kuruRN7h6qlqtK3bl+61uyKiWH+KkAqilL06TKQ9AImAsellB8JIaoD06SUPXTT1cL3KgeStDSxsZxfcoQzfvoYPXuEQ8BCLCoWx6xzJ0p16JDjk/RZ8lmlLY5Vux14LgcDIwB2Bdxi5BofrMsUZ9n7jahQ2pD91/azPGg5fnf9KGlYku61utPHtg+VTDOrWaYoyqtMpUhJ43UJJCkirz5m53xf4qLjsY8+hPmZ9dpFeldXSnXqmO2T9Fk68yds/wxsO2vrwOtr7xI7ffUBQ5adwdhQn6XvN8KuorZd37u+rAxcyZ5re5BI2lRpQ3/b/tQvV1/dPqworwldjkisgTlAc7Qp3Y8Bo6SUEbro6IvwugUSgNjH8exZFMCN4EfYNSiF7bNTxGzfluZJ+taU6pz8JL2RUe4aPfE77P4KHDyh23zQ067D/Hs7mkGLTxPzLJH/DWxIsxoWqbvcjrnNmktr8A725nH8Y+zM7ehv2592Nu3U7cOK8orTZSDZC6wClid/1B/oJ6V0L3AvX5DXMZAAaJI0nNh4GZ994VSobkbbofXQjwjRLtJv30HS/fvolSpFqbYelOrUCRNX15xHC0emw4HvoMFA6DgLkp+8v/noKYMWn+ba/Vhm9Haio2PFdLs9TXzK1stbWRm0kitRV7AobkHvOr3pVbsX5sXNC+sSKIpSiHSaa0tK6ZzTZ0XZ6xpIUoScjeTA8ksYFdOn7Yf2VKxZOnWRPmrrFqL37UfGxlJmwADKf/VlzsHkwPdwZBo0+hDa/wLJ20fFJjDkrzOcvfaQiR3sGPxWtQy7Sik5cfMEy4OWc+zGMYz0jOhQvQP9bPtRp2zuUuwrilI06DKQ7AOWAquTP+oDvC+lbFPQTr4or3sgAbh/4wk75/sTfT+O5r1q4tDaOjVgaGJjuTNjJg9XrMBy1EgsPvoo+8akhD1fw4m50GwkuH+bGkziEpIYteYCuy9GMqxVdca1rYteFkWyrkRdYVXQKrZc3sLTxKc0rtCY/nb9aWndUt0+rCivAF0GkirAXKAp2jWSf4CRUsrruujoi/AmBBKAZ7EJ7FsaRJjfPWo3Lk/rfnUxTE5JLzUabn35JVGbt1Bh8iTKvPde9o1JCTvGwJlF0Go8uH2Z+lWSRjJpSwArTl6nW/1K/NzDESODrAND1LMoNoRsYFXQKiJjI6lSsgp9bbW3D5cw1NHT+oqi6Fyh3rUlhBgtpfwtXz17Cd6UQAIgNZKzO8M4ve0q5pVMaT/MATNLbUp6mZBAxIhPeHLkCJVmzqBUu3bZN6bRwNZP4MIKeGcyvPXpf8eRkt8PhjJ9TzAtalnwR/+GmBbLPh9YgiaB/df3syJwBb53fTE1NNXePly3D9YlrQt45oqi6FphB5LrUsoq+erZS/AmBZIUYf732LckEAD3D+pRtZ52wVvz9CnXPxjCU39/qvxvPiWaNcu+IU0S/P0hBKyHdlOhSfppsXVnwvlyoz92VqVY7OWKZcnc1Z/3u+vHiqAV7A3biwYNbpXd6G/bn4blG6rbhxWliCjsQBIupaycr569BG9iIAGIuhvLzvkB3L/5hMadqtGwnQ1CT5AUFcW1AQOJj4ig6rKlFHdwyL6hpATw9oJL26Djb+DyfrqvD1yK5OOV5ylX0pi/BjfCxiL301WRMZGs/Xct64LXEfUsCtuytvS3094+bKSfy9uWFUUpFGpEksabGkgAEuKTOLj8EiFnIqnbpAJuA+qip69HQuQdrvXtiyY2lqorV1CsevXsG0qMh7X9tMkeu80Hp/RrLBeuP2Tw0jPoCcGS911xtC6dp34+TXzK9ivbWRG4gstRlzE3Nqd33d541vZUtw8ryktS4EAihIhGu7ie4SuguJQyxwIZQoh2wCxAH1j0fA0TIYQXMA24kfzRXCnlouTvfgE6AHrAXmAUYAocTdOENbBCSjk6u368yYEEtOsZZ3eEcXrrVWrUt8T9g3roG+gRHxZGWL/+CCMjbFavwrBChewbSojTpp8POwo9/gT77um+vnz3CYMWn+ZBTDzz+jWgdZ1y+erriVsnWBG4gqM3jmKoZ0iH6h3ob9tf3T6sKC/YS0+RIoTQB4IBdyACOAP0kVIGptnGC3CRUo54bt9maANMy+SPjgFfSikPPbfdOeBTKeWR7PrypgeSFL77wznmHUKVemVpN8wBQyN94gIDuTZgIAYVKlB1xXIMyuRQeyQ+Blb0gIgz4PkX1O2Q7us7j+PwWnKG4Mhofu7hSI+G+V9Evxp1lZVBK1NvH25UoRH9bbW3D+vr6SD7saIo2cptICnMm/kbAaFSyitSynhgDdAll/tKwBgwAooBhkBk2g2EELWAcqQfoSjZcGpTGbf+dbke+IBtc3yJf5qIsZ0d1vPmkRAeTvjw4WhiYrJvxKgE9F0HVk7adZPQfem+LlfKmLXDmtC4elk+9/blj0OXye8fK9XMqvF1k6/Z23Mvnzf8nPDocEYeHEmXzV3wu/tK1VZTlNdaYQaSSkB4mvcRyZ89r4cQwk8IsV4IURlASnkCOAjcSn7tllIGPbdfH2CtfBOyTuqQ3VsV8Rhcj9uXo9j82wXiYhIo0bgRlWb8Spx/ABGjRiPj47NvxLgU9N8AlnVgTT+4mj6WlzQ2ZIlXIzo7VeTnXZeYsjWQJE3+/zOZFTPDy96LHd13ML3VdBI1iQzaNYhVQavyHaQURdGdwgwkmd3D+fy/+q2ATXJtk31oa54ghKgJ2KJdA6kEvC2EaPncvu/x39P2GQ8uxIdCiLNCiLN3797N5ym8nmq5lqfdcAfu34hh04zzxD6Op+Q772D17RRijh3j5pdfITWa7BspXgYGbNLWfl/VG66fSve1kYEev/V2Zshb1Vj6TxgjV18gLiGpQP020DOgrU1b1nZcS/OKzfnp9E98ceQLYhJyGEUpilKoCjOQRABpbxG2Bm6m3UBKeV9K+Sz57UKgYfLP3YCTUsonUsonwE6gScp+QggnwEBKeS6rg0spF0gpXaSULpaWOipP+xqp5mhBhxGORN19yt/TzxH9II7SPXti+flnPN6+ncgffsz5r/0SFjBwM5QsDyt7wo3z6b7W0xN83dGOCe/ast3/FoMWnybqaUKB+25WzIzZb89mVINR7Lm2h/e2vUfow9ACt6soSv7kGEiEENFCiMfPvcKFEBuTi1xl5QxQSwhRTQhhhHYEseW5ttNWYuoMpExfXQdaCSEMhBCGQKs034F2WivL0YiSO5XrlqXzqPo8jU7g7+nneHQnFvMhQyj7/vs8XLmSe/Pm5dxIyQowaCsULw3Lu8HtgAybDG1Znd96O3P++kN6/+8Et6PiCtx3PaHHEIchLPJYRHR8NH139GXr5a0FbldRlLzLzYhkBjAW7RSTNTAG7ehhDbA4q52klInACGA32iCwTkp5UQjxrRCic/JmI4UQF4UQvsBIwCv58/XAZcAf8AV8pZRpf0t4ogKJTljVMKPrp/VJjNewcfp5HtyModzYMZh16cK9OXN5uDoXl9nMWhtMDE3gry5wNzjDJl3rV2KJVyPCH8TS449/CL0TrZP+u1ZwxbuTN3bmdnx17Cu+PfEtz5Ke5byjoig6k5ukjaeklI2f++yklLKJEMJXSulUqD3UAXX7b84e3Iphy28XSEzU0HmkM5YVixPxyUieHD5MpRm/Uqp9+5wbuRcKS9prC2K9vwPKZhywBtyIwmvJGRKSNCz2cqFh1bI66X+iJpE5F+awOGAxduZ2/NrqV5W/S1EKSJe3/2qEEJ5CCL3kl2ea79QtM6+JslYl6DamIcWKG7Bp5gVuhcVQ6beZFG/QgBtfjOPJ8eM5N2JRU7tmkvgMlnWGRxkTRNtXMuPvj5pRtoQRfReeYm9gZCYN5Z2BngGfNvyU2W6zCX8cjuc2Tw6FH9JJ24qiZC83gaQfMAC4k/waAPQXQhRHO3WlvCbMLIvT7fOGmJYuxtbZPkRcjqHyH/MoVr06EZ+M5KlfLp7dKG8HAzZC3GNtMHl8K8MmVcxNWD+8KXUrlGTY8rOsOqW7igRuVdxY22kt1qbWfHLgE2aem0miJlFn7SuKklGhPdlelKiprbyJfRzP1jk+PLgZg8eQelSpBNf69kPz5Ik2L1eNGjk3En4GlneFUhXBaweYZrxzLjY+kY9XnufQv3cZ1aYWo9+ppbPMv8+SnjH19FTWB6/HpbwL01pNw6K4Rc47KoqSSmdTW0II6+Q7tO4IISKFEBuEEGry+TVmUsqIrp/Wp1zVkuxeeJErV5Ko8uciMDDg+gdDSLiVcZSRQWVX7RPwj8K1ASX2QcbjGBmwcKALPRtaM2t/CF9t9CcxKYfnV3KpmH4xJjWdxA9v/UDAvQB6be3FmdtndNK2oijp5WZqawna23Yror1za2vyZ8prrJiJIZ1GOlOxVmn2LQ0iOEyfKgsXoHnyhOsfDCHx4cOcG7FpDn1Wwb0Q7a3BcVEZNjHU12NaT0dGuNVk9elwhq84z9P4gj24mFbnGp1Z1WEVpoamDNkzhD/9/0QjdROsFEXRyk0gsZRSLpFSJia/lgLqCb83gJGxAR1HOGLjaMHh1cEEhptgPe93EiIiCB+Wi7xcADXe1iZ3jAyAlb3g2ZMMmwghGNO2Dt92qcf+S5H0W3SShzE5pGnJg1plarGm4xrcq7rz2/nfGHVwFFHPMgY1RVHyJzeB5J4Qor8QQj/51R+4X9gdU4oGA0N92g2zp6ZLOU78fZmA2xZUnDGDuIAAIkaOyjkvF0CddtBzsTZj8Or3IOFpppsNbGrDvL4NCLj5mB7z/yHiYazOzqOEYQmmtZzG+EbjOXbjGL239ebi/Ys6a19R3mS5CSSD0T4AeBttAsWewPvZ7qG8VvT19XAfXA/b5lac3RGG34MqVPj2O2KOH+fm+PHIpFxMRdl1gW7/g7Bj2kSPiZk/NNjewYrlgxtxN/oZ3ef9Q+DNxzo7DyEE/Wz7sbTdUpJkEgN2DGDdv+tU4kdFKaAcA4mU8rqUsrOU0lJKWU5K2RXontN+yutFT0/g1r8uTm9XxvdAOD5PbbH4/HMe79iZu7xcAI6e0Hk2XN4P3u9rS/hmonF1c9YPb4aeEPT+3wn+uXxPp+fiZOnEuo7rcK3gyncnv2PCsQnEJuhu9KMob5r8Jm38TKe9UF4JQgia96qJy7s2BB6/xQXRhNLvD+bhqlXc+z0XebkAGgyEd6fDv9vh76GgyXw0U6dCSf7+uBkVzIzxWnyGbX43M90uv8oYl2Fem3l87Pwx265so9+OflyNuqrTYyjKmyK/gUQ3N/srrxwhBI07V6dp9xqEnL3DeVMPTLv24N7cuTxYtSp3jTQaCu7fwcWNsPn/IIuU9RVLF8d7eFOcKpvxyeoLLDmu21/0+nr6fOT0EfPd53P/6X3e2/Yeu8J26fQYivImyG8gUZPKb7gGHlVp1ac2Yf73OV++O8Zu7kR+9z1R27fnroHmI6H1V+C7GrZ/BllMjZU2MWL5B41xty3PlK2B/LQzCE0BimRlplnFZqzrtI5aZWox9vBYpp6eSkIW026KomRkkNUXQohoMg8YAiheaD1SXhn2rawxLKbP/mVBJFQdgFPDJ9wc/yX6ZqUxfat5zg20+gISn8KxmWBgDO1+gkyebDc21OeP/g35ZnMA/zt8hVuP4hjbtg6Vy5ro7FwqlKjAknZLmHluJssDl+N/15/praZjZWqV886K8oZTKVKUArt84Q57Fl2kTHljnC7MRlwLpuqSxRR3ykViaClh15dw6g946zNo802mwUS7qWTugVBm7AtGSmhe0xxPl8q0rVcBY0N9nZ3P3mt7mXh8IoZ6hvzU4ifeqvSWztpWlFdJblOkqECi6MT1i/fZOd8fUzNDnM7+itHDm1RdtTJ3ebmkhG2j4dxScJugHalk48ajp2w4F4H3uXDCHzylpLEBnZ0q4ulSGUdrM53k6wqLCuOzw58R+jCUYU7DGO44HH093QUrRXkVqECShgokL8bNkIds+90PY2OB46nplJDR2KxaiWHFijnvrNHA5o+1aybu32nXUHLcRXLy6n3Wn41gR8At4hI01C5viqdLZbrWr4SFabECnc/TxKd8f/J7tlzeQhOrJvzc8mfKGuumfoqivApUIElDBZIXJzLsMVvn+KCPBqfT0zAzlVRdtRKDMmVy3jkpEaLpQ08AACAASURBVP4eor2b693p2ru7culxXALbfG+x7mw4PuGPMNATtLEtR6+GlWldxxID/fzdVyKlZGPoRn44+QOljUvza6tfcS7nnK+2FOVVowJJGiqQvFj3bzxh8ywfNPEJOJ2azv+3d+bxUVb3wv+emez7HrIvLLKYgIJssgkKSkBlFyteblu1Xtveq9VPX9/27W1t77XXvbda16oVCwguKEEFFRBBBIJAwk7Ivmeyr7Oe949nEoYQwmSf4Pl+PvOZZ57nPM+ck8k83znb74TF+hL/1lvo/XyvfLLVDJv+RZtncvtftXkn3eRceQObDxfx4fdFGBpNhPt7svT6GFZMjGNEhF8PSgSnq0/zyO5HKG0s5eGJD7Nm7Jo+C3mvULgqSiQOKJEMPLXlzXz8lyMYG4ykHnqe6LHhxL7yCjoPjyufbDHChtVwficsfU2bEd8DzFYbu05XsCmjiF1nKrDaJBMTglkxMZa01Cj8vdy7db16Uz3/b+//Y2fhTm5JuIUnpj+Bn0fPxKRQDAWUSBxQIhkcGqpb+fiFIzQamkn5/q8kTEsi5plnEHonOq1NzbB+JeR/Cyve0mJ19YKKhla2HClmU0YR2RWNeLvrWZgSxcpJsUxOCnG6diGl5J2T7/D84eeJ9Y/l2dnPck3INb3Km0LhqiiROKBEMng01RnZ+r9HqSlp5NrMVxgx/1qG/e53zt24jY3w7lIoPgx3rYdRC3qdHyklRwtr2ZRRxNZjJTQaLSSE+rBiYizLJsYSFejcFKnD5Yd57OvHqDfV89upv+XOEXf2Om8KhauhROKAEsng0tpkZutfj1GZV8fYk28xbvkUwn/5CydPrtPWfq84BXe/B8Nv6rN8tZisfHZc66D/LqcanYCZI8NZMSmWW8ZG4unWdc3J0GLg13t+zcGygywduZTHJz+Ol5tXn+VPoRhsXEIkQohbgb8AeuANKeWfOxxfCzwNFNt3vSilfMN+7CkgDS2MyxfAv0sppRDCA3gRmAPYgN9IKT/oKh9KJIOPqdXCtpeOUXKultFn1nPdj28iZM09zp3cXA1vL4LqHLjnA23lxT6moKqZ9w8X8v7hIkrqWgnycefOCTGsmBTLuOjAy55ntVl56ehLvJ71OtcEX8Nzc54jPiC+z/OnUAwGgy4SIYQeOAvcAhQBh4DVUsqTDmnWApOklD/vcO50NMHMsu/aCzwupdwthPgDoJdS/lYIoQNCpJRdxhlXInENzCYrn7+SScHJGkZmv8+U/0gjcFGacyc3VsLbC6G+BNZs0daE7wesNsm+bAObDxex/UQZJouNsVEBrJwUyx0TYgj27XywwJ6iPTz+zePYpI0/3fgn5iXM65f8KRQDiSuIZBrweynlAvvrxwGklE86pFlL5yKZhlbrmIEW22sPsEZKeUoIUQiMllI6sc6rhhKJ62A129jxRiY5x6pJyt/GzMfvxH/WTOdOri+Ft26DhjJImgUJ0yDhRoiaAG5OjAbrJrXNJj45VsKmjEKOF9fjoddxy7hIVkyMZebIcPS6i/t5ShpL+NXuX3G86jhrx63ll9f/Endd90aGKRSuhCuIZDlwq5Typ/bXa4ApjtKwi+RJoBKt9vKwlLLQfuwZ4KdoInlRSvkbIUQQkAVsRmvaOg/8XEpZ3lVelEhcC5vVxldvZnH2cBXxJbuY94el+Fx3nXMn1xXBnmcgfx8Yzmr73LwhdhIkTIf4aRA3GTycmLPSDU6W1LP5cCFbjhRT02wmKtCLZdfHsnxiLIlhF97LZDXx9KGn2XhmI9dHXM9Ts54i0jeyT/OiUAwUriCSFcCCDiKZLKX8hUOaUKBRSmkUQvwMWCmlnCuEGIHWt7LKnvQL4NfASTTpLJdSfiCEeAS4Tkq5ppP3vx+4HyA+Pn5ifn5+v5RT0TOkTbL7H5mcPFBFbOUBFvx5OV4jR3bvIo2VULBfe+R/C2WZIG0g9BA9QZNKwo0QPxV8+ia0idFi5atTFWzOKOTrs5XYJExOCmHlpDgWpgzDx0MLqL0tZxt/2P8HvN28+Z9Z/8PUqKl98v4KxUDiCiK5YtNWh/R6oFpKGSiEeAzwklL+0X7sd0ArWr9JI+AvpbQJIeKAz6WU47rKi6qRuCZSSva9c5Rj+2uIqssk7anleMbF9vyCrfVQdFCTSv5+KM4Aq0k7FjHWLpbp2iPAifhfV6CsrpUPvi9ic0YheVXN+HroWTw+mhWTYrk+Ppicuhwe2f0IefV5PDThIX6a8lN0oqdLACkUA48riMQNrblqHtqorEPA3VLKEw5poqSUpfbtJcCvpZRThRCrgPuAW9Gatj4HXpBSbhVCbARek1LutDeNpUkpV3SVFyUS1+a7dRkc3ldPZNNZFj+7HM+IsL65sLkVSr63i+VbKDwIpgbtWFCCVltp62cJSb5s+PorIaUkI7+GTYcK2ZZVSrPJSnK4LysnxXFbSjAvHf8zn+V+xoyYGTw540mCvIL6pnwKRT8z6CKxZ2Ih8ALa8N83pZT/JYR4AsiQUn4ihHgSuB2wANXAg1LK0/bayd/QRm1JtFrHI/ZrJgDrgCC0Zq5/lVIWdJUPJRLXJ+PtfRz4zkiYqYA7nlmKV0hA37+J1QLlWVptpcAul+Yq7ZhvxAWpxE+DyHHQg7DxjUYLn2Zqc1My8mvQ6wSzR4URm3CM9KKXCfMO49nZz5ISntLHhVMo+h6XEImroEQyNPj+7zvZf9BGiKzkzqfvwDuw71ZA7BQpwXBO67hv62epK9SOeQZC/JQL/SzR13V7ZNj5ykbeP1zEB4eLqGgwEhxchkf0PzHJGn49+desumaVCvyocGmUSBxQIhk6HHs5nX1HPAnQN7DszwvxDhzgmeK1hXap7NNqLoYz2n43L4i9wS6WaRA7GTydC9hosdr45pyBTRmFfHkmF7fI93DzP80Y/9k8NecJ4oOC0emUUBSuhxKJA0okQ4us5zay93Qwfu4mlvzXLfgFDWLYkSbDhdpKx5FhUeMvdN7HT3NqZFhVo5GPjhTx1ok3qfPairT4YW24ngDzFKJ8kgj39yIywJMI+3NkgBfh/tpzqK+HEo5iQFEicUCJZOhx/I+vsbcgDm9vWPKfcwgIdS6YYr/TPjLMLpfiw2A1asfCx1zczxIYc9nLSCl5//g3/PPMP8hpOozEipeMxaNlEk3VKdQ2XDoPRq8ThPt5EuEgmvZn+3ZEgCehvp6XTJZUKHqCEokDSiRDDyklJ/7P0+wzjMXD14Mlv5lBUGQ/95n0hCuODJt+oZ8ldHinI8OqW6vZnred9Jx0MiszEQgmRd7AzKgFjPKfTkOznooGI+X1rZTXG6loMFJR30pFg5HqJtMl19PrBGF+HkQGeLXLJbLtWQlH0Q2USBxQIhmaSIuFE//+n3zbMhm9vx93PDaFsFgXX0jKaoHy45pULjcyLH669hx57SUjwwrqC0jPSSc9J53ChkI89Z7cFHcTi4cvZlr0tEtCrpgsNiobNclU1BupaGht3y7vlnA8ibA/R3Z4DvVTwvmhokTigBLJ0MVmNHLq/kfZr7sJ6R/E7Q9PIjKpH4YG9xeXjAzbD3X20eqeAVpY/Mn3azUWh9qKlJJMQyZbz29le952ao21BHsGc2vSrSxOXsy1Ydd2a8RXm3Aq2ms1dtnYRVNe30plg5GqToSjExDuf6EZrWM/zoX9nmoU2lWGEokDSiRDG2tjI6fWPsRBvzTM/mEs+sV1xIwKHuxs9Zy2kWF5e+HUJ9BSA5EpMPVncO1ycL94cIHZamZfyT7Sc9LZVbALk81EQkACaclpLEpaRFxAXJ9lzWSxYWi8IBhH8Tg2q3UmnNHD/HlwznDSUqJw06sZ/FcDSiQOKJEMfSwGA6fvfYCM8OU0+0QSPTKIhJQwElNCCR7WtwEaBxRTM2RthgOvQMVJ8AmDSf8Kk34CAVGXJG8wNfBl/pek56RzqOwQEsn48PEsTl7MgsQFAzZrvk04bbWZwupm3jtUyLmKRmKDvXlgVjIrJsXh5d79SZ0K10GJxAElkqsDU1ER5+75Cfkh06hOmEpdizZBMDDCm0S7VKJGBKF3G4K/hqWE3D2aUM58pvWdjL0Tpj6oRTbuhLKmMrblbCM9J53s2mzcdG7MiJnB4uTFzI6bjafec0CLYLNJvjpdwcu7s/m+oJZQXw9+PCOJe6YmEOitwukPRZRIHFAiuXownjtH6X/+npbvv6fVL4LG6cuoCh9PWbnEarHh4aUnbmwoiamhJIwLxdu/79cp6Xeqc+Dg63DkXTDWQ8wkTShj7wD9pTdkKSVna86y9fxWPs39lMqWSvzd/bkl8RYWJS9iYuTEAQ0WKaXkUF4NL+/OZteZSvw83fjRlHh+PCOJyAC1FPFQQonEASWSqw/juXPUbNpM3ccfY6uvR5c4AuPNd1MZPJaCsw0015lAwLCkABJSwkhKDSMk2ndodQYbG+DoBq2WUn0e/KO0Jq9J/wq+nQe2tNqsHCw7SHpOOl/mf0mzpZlhvsNIS0pj8fDFDA8aPqBFOFlSz6t7zrP1WAluOh1Lr4/h/lnJJIe7+Og7BaBEchFKJFcvttZWGrZvp2bTZloOHwZ3d/xvvhnbzcspsw0j/3gVFfnavA6/EE97E1gYMdcE4TZU2u9tNsj+Eg68DOd3gt4TUlZonfPDLh/8scXSwq6CXaTnpPNtybdYpZUxIWNIS05jYdJCwn3CB6wIBVXNvP5NDpsyCjFZbdx27TAenD2ClNjAAcuDovsokTigRPLDwJidTe3mzdRu+RhbXR3uCfEEr1yJ29w0ikts5GUaKDxdg8Voxc1DR+zoEBJTQklMCcM3aGD7E3pM5Rk48Coc2wDmZkiYAVMegNFpXUYrrmqp4vO8z0k/n87xquPohI6pUVNZlLyIefHz8HEfmMmelQ1G3v42l3f259PQamHGiDAenDOc6cNDh1ZtcQhwvrKRQ7nV3DU5vsfXUCJxQInkh4WttZWGHTuo2bSJloy2Wso8gleuxOP6SZRm15OXVUVepoGG6lYAwuP9NamkhhEe549w9Ql4LTXw/TqtL6WuAALjYfJ9cP0a8O56aHRuXS7pOelsy9lGcWMx3m7ezI2fy6LkRUyNmoqbzq3fs9/Qamb9gQL+vjeXigYjqbGBPDh7OPPHDVOTH3tBrqGJT7NKSc8s5VRpPToBB/7vzYT79+yHkhKJA0okP1yM589Tu2kzdVu2YK2rwz0+nqAVywlasgR9aCjVJU3kZRnIz6qiLKcOKcEnwIMEe00ldnQwHl79f2PtMVYLnPlUq6Xk7wV3Hxi/Gqb8DMJHdXmqlJKjlUdJP5/O53mfU2+qJ8QrhIVJC1k0fBFjQ8b2ey3BaLHy0ffFvLonh1xDE8lhvjwwO5k7r4vB022IND0OMgVVzaRnlbAts5QTJfUATEwIJi0lioUpUQzrRQRtJRIHlEgUNqORhh1fUPveezRnZGi1lHnzCF65Ap+pUxE6HS2NJgpOVJOXZaDgRDWmFgs6N0HsqOD2OSsBYS4SPLIzSjM1oWRt1gJJDp8LUx6EETeDrutRWyariW+Kv2FbzjZ2F+7GbDOTFJjEouRFpCWnEeN3+QCUfYHVJtl+ooyXd58nq7iOyABPfjIjibunJODn6cIiHyQKq5v5NKuUbVmlZBbVAXBdfFC7PKKD+ub/VInEASUShSPGnBytlvLRR1otJS6OoBUrCFpyJ27hWge01WqjLLuOvCwDeVlV1JY3AxAS7ds+ZyUyOdA1w7o3GSDjLTj0BjSWQegImPwATFgNnv5XPL3OWMcX+V+QnpPO4fLDAFwfcT2Lhi9ifsJ8Aj37r4NcSsm+7Cpe/jqbfdlVBHi5ce+0RNbemEiY3xDpx+onimtb+DSzlPSsUo4V1gIwPjaQRanR3JYyjNjgvu/nUiJxQIlE0Rk2o5GGL76kdtMmmg8eBDc3/OfOJWjlSnynT0M4/IqvLW9ul0rpuVpsNomXrzvx14aQmBJG/NgQPH1cbNKdxaSFYPnuZSjO0GJ7XXePFtsrJMmpS5Q0lvBp7qdsPb+VnLoc3HXuzI6dzaLkRcyMnYmHvv/m6RwrrOWVr8/z+YkyPPQ6Vt0Qx30zk4kLccEo0P1EaV0Ln2aVsS2zhO8LNHmkxASSlhpFWkpUv/8tlEgcUCJRXAljTi61m+21lNpa3GNjtVrK0iXttZT2tC0WCk9qTWD5x6tobTQjdILoEYEkpmrDi10u5H1RhiaUk1vAZoVrbtP6UZJmdRraviNSSk5VnyI9J51Pcz6lqrWKAI8AFiQuYFHyIiZETOi3SY/nKxt57escPjxShE3C4tQofjZnOKOHDaHgnd2gvL5Va7bKLCUjvwaAsVEBpKVGsSg1ioTQgQsJpETigBKJwllsJhMNX3xB7abNNB84oNVSbrpJq6XcOP2iWgpoYUEq8urJy9RqK1XFjYALh22pL4GMN7VHcxVEjNWGD6euAnfn2tUtNgsHSg+QnpPOVwVf0WJpIcYvhjtH3MnyUcsJ8+58smRvKatr5e97c1h/oIAmk5W5oyN4cM5wbki88sqUrk5FQyufHy8jPbOUQ3nVSKkFwVyUqvV5DNYETiUSB5RIFD3BmJtL7eb3tVpKTQ3uMTEErVhO4NKluEdEdHpOfVUL+VlV5GVVUXymxnXDtphb4fj78N0rUJ6lDRmeuBZu+CkExjp9mWZzMzsLd/JJ9ifsL92Pm86N+QnzWT16NePDx/fLqK+6ZjPrvsvjzX15VDeZmJgQzIOzhzN3dIRr9lldBkOjkc+Oa81WB3I1eVwT6U+aXR4jIgZ/9r8SiQNKJIreYDOZaPzyS2o2bab5u+9Ar8d/rr2WMn06Qt/5MFWz0UrR6WptzkqWwSFsSyCJqdrw4kEP2yKltlbKdy9rw4gRMPZ2rdkrbopTzV5t5Nfns/H0RrZkb6HR3MjY0LGsHr2aWxNvxcut72NstZisbD5cyKtf51Bc28KoSD9+Nns4i8dH4+6iYeyrGo1sP1HOtqwS9p+vwiZheLgvi1KjSUuNYlTklQdDDCRKJA4okSj6ClNeHrXvv0/thx9hra7GPTraXktZhntk57UUAGmTGIoayc00kJ9laA/bEhLty43LRxA/NnSginB5avLh0Ovw/TvQWgdRE7RgkeOWgJvzI6aazc2k56Sz4fQGsmuzCfIMYunIpay6ZhXRftF9nm2z1ca2zFJe3n2eM+UNxAR5c9/MJFbdEI+3x+DPRalpMrH9RBnbskr59nwVVpskOcyXRalRpKVGMyrSz2Vn9buESIQQtwJ/AfTAG1LKP3c4vhZ4Gii273pRSvmG/dhTQBqgA74A/l1KKYUQu4EooMV+znwpZUVX+VAiUfQ10mSiYedOat57j+b9Wi3Fb84cgleuwHfGjMvWUtpoqjOSl2ngyI4C6ipbSEwN48ZlI1yjk97UpIVgOfAqGM5qSwTf8BOY9GPwu7wsOyKlJKM8g/Wn1rOzcCcAc2LnsHrMaqYMm9LnN08pJbvOVPDy7vMcyqshxNeDtdMTuXdaAkE+A9ucWNdsZvvJMrZllrIv24DFJkkI9dHkkRLNmCj//pOH1QyVZ7CUHuNs2WHG3vZcjy816CIRQuiBs8AtQBFwCFgtpTzpkGYtMElK+fMO505HE8ws+669wONSyt12kTwqpXTaDEokiv7ElJ9/oZZSVYVbdBRBy5cTtGwZ7pGRXZ5rNds4tquQjE/zsJptpM6NY9LCRDy9XWASns0GOTs1oZzbAXoPGLdUCxYZfV23LlXaWMrms5t5/+z71BhrSA5MZvXo1Swevhhf974fhZSRV80rX5/ny1MV+HjouXtyPD+ZmURUYP9NKK1rMfPFyXK2ZZawN9uA2SqJC/HWmq1SohgXHdD38mitg/ITUJYFpZlYyo5xsCGXHd4efOXjTaNOx+47txEYlNCjy7uCSKYBv5dSLrC/fhxASvmkQ5q1dC6SacCLwAxAAHuANVLKU0okCldFq6XsonbTezR9ux90OvzmzCFo5Qr8Zs7sspbSVGfkwMc5nNpfirefO1PvHM7oaVGu03lsyIaDr8LR9WBqhLipmlBGLwa989IzWo1sz9vO+lPrOVF1Al93X+4Yfgd3jb6LpEDn5rZ0hzNlDbz69Xk+PlaCTsCdE2J4YPbwPuvIbmg18+WpcrZllrLnrAGT1UZMkLe92SqKlJjAvpGHlNqIu7JMTRptzzV5WICDXl7sCAzmKx9ParHho/NgzrDJzB+5lJlxs3s838cVRLIcuFVK+VP76zXAFEdp2EXyJFCJVnt5WEpZaD/2DPBTNJG8KKX8jX3/biAUsAIfAH+SnRRCCHE/cD9AfHz8xPz8/H4pp0LRGaaCAmo3v0/thx9qtZSoKIKWLSNo+TLchw277HkV+fXs3XSO0vN1hMX5MXPlKKJHDszyuU7RWqctuHXgVajNh4AYbaTXxLXg071huFmVWaw/vZ7tedsx28xMi5rG3WPuZmbMTPRdRDLuCYXVzfx9by4bDxVgtNiYPzaSB+eMYEJc9/+2jUYLX50qJz2zlK/PVmKy2IgO9NImCaZGMz62l/KwWrQmRUdhlGVBS3V7EnNIEofCk9jhIfiqtZhaSzM+bj7MiZvD/MT53Bh9Y58McHAFkawAFnQQyWQp5S8c0oQCjVJKoxDiZ8BKKeVcIcQItL6VVfakXwC/llLuEULESCmLhRD+aCJ5V0r5Tld5UTUSxWAhTSYadu2mdtMmmvbt02ops2dfqKW4XfprXkpJ9uEKvv0gm8YaIyMmRTB96Qj8Q1xodUGbFc5u19ZIyd0Dbt4Qc702dDgwVhNMYBwExmivvS4fVsXQYuDDcx/y3pn3qGiuIMYvhlXXrGLJiCV9vgZ9VaORf3ybxz/251PXYmZacigPzhnOzJFhXd78m4wWdp6uYFtmKbvOVGC02BgW4MXCFK3mcV1cUM9qj8aGC01TbdIoP6nFSgNt7ZnIsTAsBXPkOA55urOj4TxfFX9DrbG2XR4LEhdwY8yNfb68siuI5IpNWx3S64FqKWWgEOIxwEtK+Uf7sd8BrVLKpzqcs5ZOmsY6okSicAVMhYUXaikGA27DhhF8112E3LsGnc+lnexmk5UjOwo4sj0fCVw3P57r5yfg7jn4I5EuovyENsGx/CTUFUFDCdgsF6fxDLDLJfaCXALjLuwLiMGi07GrcBfrT60nozwDT70naclp3HXNXYwJHdOnWW40Wth4sIA3vsmlrL6VcdEBPDhnOLddG9Uexr7FZGXXmQrSM0vYebqCVrONCH9PFqZoM8yvjw92Xh5SQkPZpbWM6hzAfg/2DoZhqdpiZfZnc0gShyqOsD1/O18VfEWdsa7f5eGIK4jEDa25ah7aqKxDwN1SyhMOaaKklKX27SVotY6pQohVwH3ArWhNW58DLwCfAUFSSoMQwh3YAHwppXylq7wokShcCWk207BrF7Ub36Pp22/Rh4cR/tBDBC1bhnC/NF5XQ3Ur+z/M5lxGBX7BnkxbOpyRkyJddsgoNis0lmtSaXvUF9u3C6GuGJoNl57nF9kulbM+gWy0VpLecI4Wm5nrQsexeswabk6cj3sn69b3FJPFxpYjxbyy5zw5lU0khPpw1w3xnCip46tTFbSYrYT5ebIwZRhpKVFMSgy58nopNitUZV8sjdLMi8scnHiRMBiWCgHRIARmm5mDpQfZkb+jXR6+7r5as1XC/H6XhyODLhJ7JhaiCUAPvCml/C8hxBNAhpTyEyHEk8DtgAWoBh6UUp62107+hjZqSwKfSykfEUL4onW8u9uv+SXwiJTS2lU+lEgUrkrz90eoeOYZWr7/Ho/ERMIfeRj/W27pVBIl2bXs3XSOyoIGhiUHMnPVSCIShmi8KXOLJpT6NtkU2yXjIB1zM/U6wRY/PzYG+FHo7k64DVaIQJb7jyI8KOlCU1rbw4noxp1hs0l2nCzn5d3ZHCuqI9TXg9tShpGWEs3kpC7kYWrSamJlmRc3TVnssxN07hAxRhNFlF0akeMuaepzJXk44hIicRWUSBSujJSSxl27qHj2OUznz+M9fjwRjz2Kz6RLv7/SJjm1v5TvPs6hpcHE6GlRTL0jGd/AqyzEupTaKpD2Go2troh9FRlsqDvJN9Y63KTklqYWVtfXM8Foov027xnYofksFgIcRBMQDV3UaKSUFNW0EBXohVvH2fGNFZosSh2apqqyaW+a8gq01zDaahkpEDYK3DofMdWVPBYkLGB6zPRBkYcjSiQOKJEohgLSYqFuyxYq//evWCoq8LvpJiIeeRjPkSMvSWtqsZDxWR7HvipE76Zj0sJExs+NQ+/umqFB+pKC+gI2ntnIlnMf0WBuZIxfPKtDJnCbPhiv9iY1exOaw0gnDQH+wxz6a2IvHSDgE6L1XTj2ZZRlac11bQTFXyyMYSnauVdobmyTx/a87ews3Nkuj5vibmJ+wnyXkIcjSiQOKJEohhK2lhaq171L1euvY2tqInDJnYT/4hedDhuuLW9m3wfZ5GUaCAj35sZlI0ga3/UIpKuFZnMz23K3sf7UerJrswn0DGTpiKWsGr3qwoqOpiZt/kVbs1mdvdms3qH/xtLa4cqC9lqGzg3Cx1wsjGHXah3jTmK2mTlQeoAdeTuckoeUEqvZhtlovfAwac8Wx31GKxZT27YNs9Fif3bcb2X17yaj62HsMSUSB5RIFEMRS00NVa++Rs0//wk6HSH3riH0vvvQB1zaL1J4sppvNp+jprSJ2NHBzFgxktCYwY8eOxC0hWLZcHoDOwt2YpM2ZsfNZvXo1UyLmta1VKWE5mpNNG19M40V2sJfw1Ih/JrLxhmTNnnhBt/hht7aauJ0+VlOlJ0iuzIHq0niix/x3onEeMYR7BaK1SSxGK2YHAVh0ra7c1vWuQncPfW4e+i1Z/vDzf48794xuPUw5pgSiQNKJIqhjKmoGMNf/5e6T7aiCwgg7IEHCP7RSmAaagAAE29JREFU3eg8L77B2aw2ju8p4eDWHEwtFq6dFcPkxcl4+bnYyo39SFlTGZvObOKDcx9Q3VpNYkAid42+izuG34GfR9ditVptlJyrpehUDa3NZsytF/+y7/hL32KydStvencdHl563Dre8Du8vvx+He6ebrh5aM/unjrcPPXou6htSCl7VTtVInFAiURxNdB6+jQVzzxL0969uEVHEf7LXxK4ePEloVdaG80cTM/l+J5iPLz0TF6cxLhZMV3ecK42TFYT2/O2s/H0RjINmfi4+XD78NtZPXo1yUHJF9K1aqtd5hyrJD+rCmOzBZ1e4OXr3v6L3t1Dj7vXhV/8F/br0HkIiloLOFGfRWbNMeplDe4eeibEjOfGhGlMibsBHx8v3Dz0AxruxlJZSc3G92j85hsS1/+z04mvzqBE4oASieJqomn/fiqeeZbWEyfwvOYaIn71CL4zZ17yy7OquJG9m89RdLqG4ChfZq4YSdzYob+aYHc5bjjOhtMb+Cz3M8w2MzcGz2I+y/AoCKHodC1Wiw0vX3cSU0NJGh9O3NgQ3LtoCjJbzXxX+h078news2An9aZ6/Nz9tD6PxPlMj57er2vZd0VL1nGq171D/Wefg9mM7+xZRP/3f+MW2rNlCpRIHFAiUVxtSJuNhs8/p+L5FzAXFuIzZQoRjz6Kd8q1F6eTktxjBvZ9kE19W7j65SMIinCBcPUDSF1lM8czCjh28Dy2Ui8EgmavOvyvgTkzJzJqTGyXHdKuLA9pNtPwxRdUv7OOlqNH0fn4ELh0KSH3/AiPxMReXVuJxAElEsXVijSZqHlvE4a//Q1rTQ3+t91KxH/8Bx4JF4cNt5ptHNtpD1dvsTHeHq7ewxXC1fcDUkoMhY3kHK0k52gl1SVNAITF+ZGQGoohMpePqjdwsPwgHjoPFiYvZPXo1YwNHdt+jTZ5tA3VbTA1tMtjQeICpkVPGzR5gDYYo3bTZmrWr8dSXo57fDwh9/yIwCVL0Pv3zUqLSiQOKJEornasjY1Uv/kmVW+9jTSbCV61irB/e/CSJo3OwtWPmRaFcJVw9b3AarVReq6WnGMGco9W0lhjRAiIGhFE8oRwksaHERB28Xok2TXZbDi9ga05W2mxtDA+fDxpyWmcMJy4SB5z4+cyP2H+oMsDoPXMGarXraN+azrSaMRn2lRC1tyL3+xZV1xQrbsokTigRKL4oWCprKTypZeo3fw+Ok9PQn7yY0LXrkXne/HiURX59Xzz3jnKcuoIj/dnxsqRRI9woXD1TmI2Wik4WUXuUQN5WQaMzRb07jrix4aQND6cxNRQvP2ufOOvN9XzSfYnbDyzkfz6fJeTh7Raady1i+p31tF88CDCy4vA228nZM09nU5Y7SuUSBxQIlH80DDm5lL5/As07NiBPiyM8If+jaDlyy8KCiml5FxGOfs/PE9jjZGRkyKY5mrh6juhpcFEbqaB3GMGCk9VYzXb8PR1IykljKQJ4cSNCelxhGSbtJFTm0N8QPygywPAWl9P7fsfUPPPf2IuLsYtKoqQH91N0PLl6IP6X/xKJA4okSh+qLQcPUrFM8/SnJGBR0IC4Q8/jP+C+ReN8DIbrRzZkc/3OwoQaOHqr1uQ0OXIpYGmrrKF3GNaf0fZ+TqkBP8QL5ImhJE8PpyoEYE9nr3tihhzcqh5911qt3yMbG7Ge9JEQu5Zg//N83o8lLcnKJE4oESi+CEjpaRx924qn3sO47lsvFJTiXj0V/hOnnxRuo7h6qcvHcGISRGDEm6lvbP8WCW5RyupKtY6y0Nj/Uger9U8wmL9rqpQMNJmo2nvXqrfWUfT3r0Id3cC0tIIXnMP3uPGDUqelEgcUCJRKLR29rotH1P5179iKSvDb/Zswh95BK9rRl2UruRcLd9sOouhsJGo4YHMWDkw4eptVhsl2XXkHq0k51gljdVX7iy/GrA2NlG3ZQs1776LKS8PfXgYwatXE7xyJW5hYYOaNyUSB5RIFIoL2FpbqXn3XQyvvY6toYHAO+8k/Bc/xz06+kIam+T0/lK+23KelkYzY6ZFMaUfwtWbjdb2meV5WQaMTT3rLB+KmAoLqXn3n9R+8AG2xka8UlMJWbOGgAXzER6uUWYlEgeUSBSKS7HW1mJ47XVq3n0XgOA19xB2//3oAy8sumRqsZDxaR7Hdhaid9cx6bbeh6tvaTCRl2Ug56hDZ7mPG4mpWn9H3Nied5a7OlJKmg8coPqddTTu2gV6PQELFhCy5h68J0wY7OxdghKJA0okCsXlMZeUUPm/f6Xu44/R+fsT9sD9BN9zz0VBITuGq5+xfASJqc6Hq683tJBztJLcYwZKs2uREvxCPEkeH07ShHCir7LO8o7YWlqo27qVmnXvYjx3Dn1wMEGrVhK8ejXukZGDnb3LokTigBKJQnFlWs+coeK552j6eg9uUfagkLdfHBSy4GQVezdnU1PaRNyYYG5cMZLQ6Euj6kopMRRpM8tzjxqoKm4EIDTGr32kVVjc1dVZ3hnm0lJq1m+gdtMmrHV1eI4erTVfLUq7JHqzK6JE4oASiULhPE0HDlLxzDO0ZmXhOWqUFhRy1qz2m77VauPEnmIObs3F1Gq1h6tPwsNLT2l2nX2klYGG6tb2zvKk8WEkjQ8nMPzq6yzviJSSliNHqH5nHQ1ffAFS4j9vHsFr7sHnhhuGlDyVSBxQIlEouoeUkobt26l4/nnM+QX43HADEY89indqanua1kYzB7fmaOHqvd1A0N5ZHjcmhOQJYSSmhOHt7xodx/2NzWSi/tNPqVn3Lq0nTqALCCBo+XKC774bj9iYwc5ej1AicUCJRKHoGdJspmbzZgwv/Q1rVRX+CxYQ8fB/XBRVtqq4kUPb8nBz15E84eruLO+MtrU/at57D6vBgMfw4YSsuYfA229H5zO0oywrkTigRKJQ9A5rYxPVb71F1VtvIU0mglYsJ/yhhwZ9nsNg0pKVpQVPtK/94Td7NsH3rsF3+vQh1XzVFS4hEiHErcBfAD3whpTyzx2OrwWeBortu16UUr5hP/YUkAbogC+Af5cOmRVCfAIkSykvXoChE5RIFIq+wWIwYPjb36jZtBnh4UHo2rWE/PjH6P18r3zyVcAla3/4+mprf/zo7l6v/eGKDLpIhBB64CxwC1AEHAJWSylPOqRZC0ySUv68w7nT0QQzy75rL/C4lHK3/fhSYDmQqkSiUAw8prw8Kl74Cw2ff44+NJSwf3sQ/3nz0Hl7o/PxuSg45NWApaaG2vc2UbNhw8Vrfyxdit6v67XghzLOiqQ/o39NBrKllDn2DG0E7gBOdnmWhgS8AA9AAO5Auf06fsAjwP3Apr7PtkKhuBIeiYnEvvA8LZn/SsUzz1L+xz9R/sc/tR8X7u4IHx90bQ+7YHTe3uh8fbRj3h2O+2rboj2tT/u+wRJU65kzVL/zjrb2h8mE7/RpDPv9f+I3ezZCd/XOe+ku/SmSGKDQ4XURMKWTdMuEELPQai8PSykLpZT7hRC7gFI0kbwopTxlT/9H4Fmguas3F0LcjyYb4uPje1UQhULROd6pqcT/422aDx3ClJ+PbG7G1tKCrcn+3NysPVqakU3NWCorsRU47G9uBqvV6fe7SFBtwnFGUN7eDud1LahO1/5YskRb+2PEiP74Mw55+lMknfU2dWxH2wpskFIahRA/A/4BzBVCjADGALH2dF/YZVMPjJBSPiyESOzqzaWUrwGvgda01eNSKBSKLhFC4Dt58iXRhJ1BSok0mzUBtUvHLiK7gGzNzcg2KV1OUAYDtgLH85u6JSjc3dvlI00mrFVVuEVHEfHYowQtWzYga38MZfpTJEVAnMPrWKDEMYGUssrh5evA/9i3lwDfSSkbAYQQnwFTgQZgohAiDy3vEUKI3VLKOf1RAIVC0b8IIbQAhR4efXqz7lJQLRf2dSYoaTHjP+9m/OfNHdC1P4Yy/flXOgSMFEIkoY3Kugu42zGBECJKSllqf3k70NZ8VQDcJ4R4Eq1mMxt4QUq5FXjZfm4ikK4kolAoOtJfglJ0Tr+JREppEUL8HNiONvz3TSnlCSHEE0CGlPIT4JdCiNsBC1ANrLWf/j4wF8hCaw773C4RhUKhULgYakKiQqFQKDrF2eG/avyaQqFQKHqFEolCoVAoeoUSiUKhUCh6hRKJQqFQKHqFEolCoVAoeoUSiUKhUCh6xQ9i+K8QohLI7+HpYYChD7MzmFwtZblaygGqLK7K1VKW3pYjQUoZfqVEPwiR9AYhRIYz46iHAldLWa6WcoAqi6tytZRloMqhmrYUCoVC0SuUSBQKhULRK5RIrsxrg52BPuRqKcvVUg5QZXFVrpayDEg5VB+JQqFQKHqFqpEoFAqFolcokdgRQtwqhDgjhMgWQvyfTo7/TAiRJYQ4KoTYK4QYOxj5vBJXKodDuuVCCCmEcNmRKU58JmuFEJX2z+SoEOKng5FPZ3DmcxFCrBRCnBRCnBBCrB/oPDqLE5/L8w6fyVkhRO1g5PNKOFGOeCHELiHEESFEphBi4WDk0xmcKEuCEOIrezl2CyFiO7tOj5FS/uAfaOulnAeSAQ/gGDC2Q5oAh+3b0dZIGfS8d7cc9nT+wB7gO2DSYOe7F5/JWuDFwc5rH5VlJHAECLa/jhjsfPfmf8wh/S/Q1iIa9Lz34DN5DXjQvj0WyBvsfPeiLJuBf7FvzwXW9WUeVI1EYzKQLaXMkVKagI3AHY4JpJT1Di99uXT9eVfgiuWw80fgKaB1IDPXTZwty1DAmbLcB7wkpawBkFJWDHAenaW7n8tqYMOA5Kx7OFMOCQTYtwPpsFS4C+FMWcYCX9m3d3VyvFcokWjEAIUOr4vs+y5CCPGQEOI82k34lwOUt+5wxXIIIa4D4qSU6QOZsR7g1GcCLLNX198XQsQNTNa6jTNlGQWMEkLsE0J8J4S4dcBy1z2c/VwQQiQAScDOAchXd3GmHL8H7hFCFAGfotWuXBFnynIMWGbfXgL4CyFC+yoDSiQaopN9l9Q4pJQvSSmHA78Gftvvueo+XZZDCKEDngd+NWA56jnOfCZbgUQpZSrwJfCPfs9Vz3CmLG5ozVtz0H7FvyGEcMXFxp36rti5C3hfSmntx/z0FGfKsRp4W0oZCywE1tm/Q66GM2V5FJgthDgCzAaK0ZY47xNc8Y8yGBQBjr9mY+m6GrsRuLNfc9QzrlQOf+BaYLcQIg+YCnzioh3uV/xMpJRVUkqj/eXrwMQBylt3ceb/qwj4WEppllLmAmfQxOJqdOe7cheu2awFzpXjJ8AmACnlfsALLXaVq+HMd6VESrlUSnkd8Bv7vro+y8FgdxS5wgPt12AOWjW8rbNqXIc0Ix22FwMZg53vnpSjQ/rduG5nuzOfSZTD9hLgu8HOdy/KcivwD/t2GFpTRehg572n/2PANUAe9rlqrvZw8jP5DFhr3x6DdnN2ufI4WZYwQGff/i/gib7Mg6qRAFJKC/BzYDtwCtgkpTwhhHhCCHG7PdnP7cMyjwKPAP8ySNm9LE6WY0jgZFl+af9MjqH1Wa0dnNx2jZNl2Q5UCSFOonWGPialrBqcHF+ebvyPrQY2Svudy9Vwshy/Au6z/39tQJOKy5XHybLMAc4IIc4CkWgy6TPUzHaFQqFQ9ApVI1EoFApFr1AiUSgUCkWvUCJRKBQKRa9QIlEoFApFr1AiUSgUCkWvUCJRKHqJEOL3QohHXSAfeUIIV5wwp7jKUSJRKBQKRa9QIlEoOkEI4SuE2CaEOCaEOC6EWOX4i18IMUkIsdvhlPFCiJ1CiHNCiPvsaaKEEHvs63IcF0LMtO9/WQiRYZ9M+QeH98wTQvy3EGK//fj1QojtQojzQoif2dPMsV/zI/vaJa90Fv9JCHGPEOKg/b1fFULo+/Pvpfhho0SiUHTOrUCJlHK8lPJa4PMrpE8F0oBpwO+EENHA3cB2KeUEYDxw1J72N1LKSfZzZgshUh2uUyilnAZ8A7wNLEeLifaEQ5rJaLOuU4DhwFLHjAghxgCrgBvt720FftSNsisU3cJtsDOgULgoWcAzQoj/AdKllN8I0VmQ1XY+llK2AC1CiF1oN/tDwJtCCHdgi5SyTSQrhRD3o33/otDWisi0H/vE4f39pJQNQIMQotUhGvBBKWUOgBBiAzADeN8hL/PQAlgesufZG3DV9U0UVwFKJApFJ0gpzwohJqKFD39SCLEDLex2Wy3eq+Mpl15C7hFCzEKrqawTQjyNVtN4FLhBSlkjhHi7w7XaohnbHLbbXrd9Xy95rw6vBVoAyMevUEyFok9QTVsKRSfYm6aapZTvAs8A16NFs20LVb+swyl3CCG87IsFzUGrDSQAFVLK14G/268RADQBdUKISOC2HmRvshAiyd43sgrY2+H4V8ByIUSEvSwh9rwoFP2CqpEoFJ2TAjwthLABZuBBtCaivwsh/i9woEP6g8A2IB74o5SyRAjxL8BjQggz0AjcK6XMtS8udAIt9Pe+HuRtP/Bnex73AB85HpRSnhRC/BbYYZeNGXgIyO/BeykUV0RF/1UohhBCiDnAo1LKRYOdF4WiDdW0pVAoFIpeoWokCoVCoegVqkaiUCgUil6hRKJQKBSKXqFEolAoFIpeoUSiUCgUil6hRKJQKBSKXqFEolAoFIpe8f8BGEdcNaYZtxgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xc459b38>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch4_1.best_score_, gsearch4_1.best_params_))\n",
    "test_means = gsearch4_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch4_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch4_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch4_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch4_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5.正则参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.1, 1, 1.5, 2], 'reg_lambda': [0.1, 0.5, 1, 2]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 0.1, 1, 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.1, 0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58392, std: 0.00385, params: {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58368, std: 0.00352, params: {'reg_alpha': 0.1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58333, std: 0.00340, params: {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  mean: -0.58370, std: 0.00391, params: {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  mean: -0.58325, std: 0.00359, params: {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58347, std: 0.00397, params: {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58329, std: 0.00369, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.58355, std: 0.00361, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.58298, std: 0.00352, params: {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       "  mean: -0.58357, std: 0.00377, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58340, std: 0.00370, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58348, std: 0.00365, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58332, std: 0.00327, params: {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  mean: -0.58344, std: 0.00362, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58363, std: 0.00340, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58369, std: 0.00369, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       " -0.5829797402919331)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5_1, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.grid_scores_, gsearch5_1.best_params_,     gsearch5_1.best_score_\n",
    "#best {'reg_alpha': 1.5, 'reg_lambda': 0.1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([355.64162474, 360.97419796, 400.66440334, 431.815236  ,\n",
       "        354.61291947, 409.9541132 , 383.47407804, 376.52287464,\n",
       "        371.64435692, 344.91213999, 344.5688199 , 340.45559993,\n",
       "        340.94927993, 346.37780004, 347.66979995, 310.82200007]),\n",
       " 'mean_score_time': array([0.68639994, 0.8402    , 1.35352001, 0.76999998, 0.69239998,\n",
       "        1.05179996, 0.86599998, 0.72540002, 0.69239998, 0.67939997,\n",
       "        0.69120002, 0.69080005, 0.70620003, 0.69680009, 0.70700006,\n",
       "        0.621     ]),\n",
       " 'mean_test_score': array([-0.58392353, -0.58367962, -0.58333381, -0.5837018 , -0.58324594,\n",
       "        -0.58347156, -0.58328639, -0.58355205, -0.58297974, -0.5835715 ,\n",
       "        -0.58339633, -0.583483  , -0.58332135, -0.58343906, -0.5836346 ,\n",
       "        -0.5836866 ]),\n",
       " 'mean_train_score': array([-0.50248226, -0.50478039, -0.50655476, -0.50894201, -0.50484835,\n",
       "        -0.50639836, -0.50795168, -0.5105142 , -0.50662489, -0.50815657,\n",
       "        -0.50953616, -0.51188107, -0.50929097, -0.51050095, -0.51155514,\n",
       "        -0.5134875 ]),\n",
       " 'param_reg_alpha': masked_array(data=[0.1, 0.1, 0.1, 0.1, 1, 1, 1, 1, 1.5, 1.5, 1.5, 1.5, 2,\n",
       "                    2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_reg_lambda': masked_array(data=[0.1, 0.5, 1, 2, 0.1, 0.5, 1, 2, 0.1, 0.5, 1, 2, 0.1,\n",
       "                    0.5, 1, 2],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " 'rank_test_score': array([16, 13,  5, 15,  2,  8,  3, 10,  1, 11,  6,  9,  4,  7, 12, 14]),\n",
       " 'split0_test_score': array([-0.57836836, -0.577774  , -0.5783291 , -0.57751771, -0.57784665,\n",
       "        -0.57734604, -0.57740823, -0.57784319, -0.57767342, -0.57720862,\n",
       "        -0.57737315, -0.57757391, -0.57765396, -0.57742734, -0.57817208,\n",
       "        -0.5776555 ]),\n",
       " 'split0_train_score': array([-0.50343845, -0.50492288, -0.507356  , -0.50994028, -0.50636385,\n",
       "        -0.50715335, -0.50938385, -0.51206917, -0.50778138, -0.50905721,\n",
       "        -0.51043668, -0.51285526, -0.5101999 , -0.5106005 , -0.5123273 ,\n",
       "        -0.51462207]),\n",
       " 'split1_test_score': array([-0.58209504, -0.58235022, -0.58079509, -0.58196905, -0.58098871,\n",
       "        -0.58100651, -0.58113161, -0.58139464, -0.58034682, -0.58194112,\n",
       "        -0.58175884, -0.5818873 , -0.58217988, -0.58139439, -0.58148977,\n",
       "        -0.58180321]),\n",
       " 'split1_train_score': array([-0.5026567 , -0.50511963, -0.50616539, -0.50944024, -0.50479634,\n",
       "        -0.50654296, -0.50803556, -0.51017396, -0.5065246 , -0.50792165,\n",
       "        -0.50974828, -0.51193142, -0.50901892, -0.51074291, -0.51148044,\n",
       "        -0.51331253]),\n",
       " 'split2_test_score': array([-0.5826316 , -0.5838383 , -0.5836519 , -0.58326413, -0.58313743,\n",
       "        -0.58371911, -0.58396452, -0.58412727, -0.58372466, -0.58414588,\n",
       "        -0.58363188, -0.58343022, -0.58399613, -0.58461983, -0.58467912,\n",
       "        -0.58439114]),\n",
       " 'split2_train_score': array([-0.50166317, -0.50416325, -0.50648201, -0.50772209, -0.50397239,\n",
       "        -0.50529164, -0.50689834, -0.51037858, -0.50624179, -0.50758217,\n",
       "        -0.50877068, -0.51100493, -0.50849323, -0.51014654, -0.51191993,\n",
       "        -0.5131625 ]),\n",
       " 'split3_test_score': array([-0.58765318, -0.58696264, -0.58731179, -0.58783544, -0.58681553,\n",
       "        -0.58726197, -0.58619261, -0.58651845, -0.58608328, -0.58708563,\n",
       "        -0.58613291, -0.58664967, -0.58574688, -0.58674075, -0.58633934,\n",
       "        -0.58633985]),\n",
       " 'split3_train_score': array([-0.50314115, -0.50543625, -0.50649688, -0.50893234, -0.50494496,\n",
       "        -0.50665362, -0.50810297, -0.51025008, -0.50647418, -0.50835505,\n",
       "        -0.50906412, -0.51201305, -0.50906408, -0.51088122, -0.51160225,\n",
       "        -0.51377738]),\n",
       " 'split4_test_score': array([-0.58887099, -0.5874741 , -0.58658217, -0.58792396, -0.58744265,\n",
       "        -0.58802558, -0.58773634, -0.58787805, -0.58707177, -0.58747746,\n",
       "        -0.58808631, -0.58787523, -0.58703102, -0.58701408, -0.58749387,\n",
       "        -0.58824466]),\n",
       " 'split4_train_score': array([-0.50151183, -0.50425992, -0.50627352, -0.50867508, -0.50416421,\n",
       "        -0.50635023, -0.5073377 , -0.5096992 , -0.50610249, -0.50786678,\n",
       "        -0.50966104, -0.5116007 , -0.50967871, -0.51013357, -0.51044579,\n",
       "        -0.51256302]),\n",
       " 'std_fit_time': array([ 9.86312991, 13.59398638, 25.41019159, 21.58534448, 12.5366202 ,\n",
       "        39.35725452,  6.81892852, 10.4571523 , 14.30374897,  9.08842661,\n",
       "         8.42231067,  4.59527445,  5.94571514,  7.88921146, 11.11917572,\n",
       "        22.34250171]),\n",
       " 'std_score_time': array([0.03068292, 0.16641675, 0.53094086, 0.0882633 , 0.02387124,\n",
       "        0.73515527, 0.23375712, 0.02881388, 0.0254918 , 0.00531403,\n",
       "        0.00825594, 0.02693263, 0.02102761, 0.03139032, 0.02954992,\n",
       "        0.05529193]),\n",
       " 'std_test_score': array([0.00385375, 0.00351694, 0.00340499, 0.00390768, 0.00359478,\n",
       "        0.00396899, 0.00368516, 0.003608  , 0.00352192, 0.0037704 ,\n",
       "        0.00370129, 0.0036547 , 0.00327174, 0.00361656, 0.00339821,\n",
       "        0.00369408]),\n",
       " 'std_train_score': array([0.0007735 , 0.00049341, 0.00041984, 0.00074891, 0.00084192,\n",
       "        0.00061371, 0.00084469, 0.00081069, 0.00059832, 0.00051374,\n",
       "        0.00057971, 0.00060253, 0.00058964, 0.00030778, 0.00062734,\n",
       "        0.00068734])}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch5_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.582980 using {'reg_alpha': 1.5, 'reg_lambda': 0.1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAELCAYAAADkyZC4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsnXd4VGXah+93Jr2TQhJIgECA0HvoVUGkSlNQqSodXXddV3dX13V3dXVdP5UiIFJFYE3oCijSQTpSQksILZSQ3idl5v3+OAMECBjCTGYS3vu6cpFz5pTnwDC/eZ8qpJQoFAqFQmFpdLY2QKFQKBSVEyUwCoVCobAKSmAUCoVCYRWUwCgUCoXCKiiBUSgUCoVVUAKjUCgUCqugBEahUCgUVkEJjEKhUCisghIYhUKhUFgFB1sbYEv8/f1lrVq1bG2GQqFQVCgOHTqULKUM+K3jHmuBqVWrFgcPHrS1GQqFQlGhEEJcLM1xykWmUCgUCqugBEahUCgUVkEJjEKhUCisghIYhUKhUFgFJTAKhUKhsApKYBQKhUJhFZTAKBQKhcIqKIEpA2euZ7H6yBVbm6FQKBR2zWNdaFlWvtoZT9ShBI4lZPB2nwgc9UqnFQqF4m6UwJSBDwc3wdPFgfm7z3PyWgYznm+Jv4ezrc1SKBQKu0J99S4Djnodf+vfiE+fbcaRS+kMmL6LYwnptjZLoVAo7AolMI/A4JYhRE/qgBCCobN/IepQgq1NUigUCrtBCcwj0ri6N2undqR1zSq88d1R/rbmBIVGk63NUigUCpujBMYC+Hk4s3hcJC93CmPRLxd54at9JGXl29oshUKhsClKYCyEg17HX/s15PPhzTl2JZ3+03dx5FKarc1SKBQKm6EExsIMbF6d6EkdcNALnpuzlxUHLtnaJIVCobAJSmCsQKNq3qyb2om2tX35U/Rx/rLqOAVFKi6jUCgeL5TAWIkq7k4sGNOGCV1rs3TfJUZ8tZcbmQZbm6VQKBTlhhIYK+Kg1/H20w2Y8XwLTl7NpN/0XRy6qOIyCoXi8UAJTDnQr2k1Vk3pgIujnuFzf+HbfSouo1AoKj9KYMqJiCAv1k7tSPs6/vx51XHeXnmM/CKjrc1SKBQKq6EEphzxcdPiMpO71WHZ/ssMn7uXRBWXUSgUlRQlMOWMXid4s3cEs15oyZnrWfT9YhcHLqTa2iyFQqGwOEpgbESfJsGsntIRD2c9I+buZckvF5BS2toshUKhsBhKYGxIvUBP1kztROe6/ryzJoY3o45hKFRxGYVCUTlQAmNjvF0d+Xp0G17tEc53hxJ4bs4vXE3Ps7VZCoVC8cgogbEDdDrB73vVZ87IVpxLymHAjF3si0+xtVkKhULxSCiBsSOeahTE6ikd8HJ15IV5+1i4+7yKyygUigqLEhg7I7yqJ6undKRb/aq8t+4kf/juqIrLKBSKCokSGDvEy8WRuSNb8fqT9Vh5+ApDZ+/hiorLKBSKCoZVBUYI0VsIcUYIESeEeKuE18cIIZKEEL+af14u9trHQogYIcQpIcQXQghh3r9RCHHU/NpsIYTevL+5EGKv+ToHhRCR1nw2a6PTCV57si7zRrXmYnIu/afvYs+5ZFubpVAoFKXGagJj/uCfCTwNNARGCCEalnDoCillc/PPPPO5HYCOQFOgMdAG6Go+/lkpZTPz/gBgmHn/x8DfpZTNgXfN2xWeJxsGsnpqR6q4OTLy6/18vUvFZRQKRcXAmiuYSCBOShkvpSwAlgMDS3muBFwAJ8AZcAQSAaSUmeZjHMyvy2LneJl/9wauPuoD2At1AjxYPaUjT0RU5R/rT/K7Fb+SV6DiMgqFwr6xpsBUBy4X204w77ubIUKIY0KIKCFEKICU8hdgK3DN/LNJSnnq5glCiE3ADSALiDLv/h3wHyHEZeAT4G0LP49N8XRxZPaLrXijVz3WHr3KkC/3cDk119ZmKRQKxX2xpsCIEvbd7dtZB9SSUjYFNgOLAIQQ4UADIARNlHoIIbrcuoiUTwHBaKubHubdk4DXpZShwOvA1yUaJcR4c4zmYFJSUlmfzSbodIKpPeoyf3QbLqfl0n/GLnbFqriMQqGwT6wpMAlAaLHtEO5yW0kpU6SU+ebNr4BW5t8HAXullNlSymxgA9DurnMNwFpuu91GAyvNv3+H5qK7BynlXCllayll64CAgDI9mK3pHlGVtVM7EeDhzKj5+5i745yKyygUCrvDmgJzAKgrhAgTQjgBw9EE4RZCiOBimwOAm26wS0BXIYSDEMIRLcB/SgjhcfMcIYQD0Ac4bT7nKrcTAXoAsVZ4JrshzN+d1VM60rtxEB/8cJppy46QW1Bka7MUCoXiFg7WurCUskgIMRXYBOiB+VLKGCHE+8BBKeVa4FUhxACgCEgFxphPj0ITieNobrWNUsp1QohAYK0Qwtl8zS3AbPM5rwCfm4XHAIy31rPZC+7ODsx8viWzt8fz8abTxN3IZu7I1tTwc7O1aQqFQoF4nF0rrVu3lgcPHrS1GRZh+9kkXl12BIAvRrSga72K6f5TKBT2jxDikJSy9W8dpyr5Kwld6wWwdmpHgr1dGLtgP19uU3EZhUJhW5TAVCJq+rmzcnIH+jQJ5qONp5ny7WFy8lVcRqFQ2AYlMJUMNycHpo9owZ/7RLDxxHUGzdrNheQcW5ulUCgeQ5TAVEKEEIzvUofF49pyIyufATN2sfX0DVubpVAoHjOUwFRiOtX1Z93UToRUcWPcogPM2BKLyaTiMgqFonxQAlPJCfV1I3pSBwY0q8YnP55l0tJDZKu4jEKhKAeUwDwGuDrp+ey55vy1bwM2n7rBMzN3cy4p29ZmKRSKSo4SmMcEIQQvd67NkpciSc0p4JkZu9l8MtHWZikUikqMEpjHjA51/Fk3rRM1/d14efFBPtt8VsVlFAqFVVAC8xhS3ceVqIkdGNyyOp9tjmX8kkNkGgptbZZCoahkKIF5THFx1PPfYc14r39Dtp7R4jJxN1RcRqFQWA4lMI8xQgjGdAxj6cttycgt5JmZu9kUc93WZikUikqCEhgF7Wr7sW5aJ+oEuDNhySE+/fGMissoFIpHRgmMAoBqPq6smNCeYa1C+GJLHC8vPkhGnorLKBSKsqMERnELF0c9Hw9tyj8GNmLH2SQGztjF2cQsW5ulUCgsTEZu+Xx5VAKjuAMhBCPb12LZ+HZk5xt5ZuZuNhy/ZmuzFAqFBbiWkcff18XQ9sPNHLyQavX7KYFRlEibWr6sn9aJeoGeTFp6mI83nsao4jIKRYXkUkoub688TpePt7L4l4v0bVKNqp4uVr+v1UYmKyo+Qd4urJjQjr+tiWHWtnPEXM3ki+Et8HZztLVpCoWiFMQmZjFr2znWHr2KXid4rk0oE7rUIdS3fMaqq5HJlWRksrX5dt8l/rb2BMHerswd1YqIIC9bm6RQKMxIKZF5eRgzMzFmZBJ37go/7DlL3LmrVDEaaFfVkWbeOpzzcjBmZGDMzCTor3/BtVmzMt2vtCOT1QpGUSqeb1uD+kGeTPrmEINm7uE/w5rSr2k1W5ulUFQqZEGBJhKZmRjTMzBmZmDKyMCYYd6XkYEpM8P8WrF9GRnIwtuBez3Qv/iF9XqKvLyQXl7ovL3R+/iAzvoREiUwilLTqmYV1k/rxKSlh5n67RGupucxvksdW5ulUNgV0mjElJV168PfmJGJMSMdk3l1oa0gMszCcFskjJmZyNzcB15b5+GB3tsbnbcXem9vnKuGo/f24qrRkd2JBZzMBjw96dKyDr3a18XL3w+9jzc6d3eEEOXzF1AMJTCKh6KqlwvfvtKW3//vKB/8cJobmfn8uU8DdLryf/MqFNZCSokpJ1dbLdwSCfOK4tbqotiKIuO2SJiysuABoQfh6orey0v78fbGMSQEl0aNzNvmFYaXN3pvb/Te2nE6b2/0np4Ih9sf2SaTZPOpRGZuO8fRy+kE1nZmfJc6jIgMxc3JPj7a7cMKRYXC2UHP9OEt8Hd3Yt6u8yRn5/Px0GY4OaikRIV9YcrPv+VCMhZbQdwhHJnmFUYxkTBmZkLRAwbzOTreIRJ6fz+c6tTWhMHLS1s13HzN2/vWcTpvb3ROTo/0TEaT5Pvj15i5JY4ziVmE+rrywaAmDGlVHWcH/SNd29IogVGUCZ1O8N6ARlT1cuE/m86QklPA7Bdb4e6s3lIKyyKLijBmZWFMN7uZisUn7nUz3RmzkAbD/S8shCYCN4XAywuHasHm32+vIDSh8NFWE+bjhJtbubucCopMrD5yhS+3n+N8cg7hVT34v+ea0b9pNRz09vnlTn0aKMqMEIIp3cPx93Di7ZXHef6rvcwf0wY/D2dbm6awM6SUmLKzzasGs5upmOvpbjdT8VWHKfvBXb51bm6aC8n84e9Uq9b93Uxe3uh9tON0np6Icgh0PyqGQiMrDlxmzvZzXM0w0Li6F7NfbEmvhkF275pWAqN4ZJ5rUwNfd2emfnuYobN/YfG4yHLLs1eUH1JKpMFwSxhKE5/QVhRml5PJdN9rC0dHdD7et1YPjoGB6OvVuxXM1sTC7GYqvqLw9EQ8osvJXsnOL2Lp3ot8tVNzQ7euWYV/DW5Ct3oBNgnYlwVVB6PqYCzGwQupvLToIE4OOhaNjaRhNVUrY4/IwsJiGU7FVxQlu5lu7Uu/MxX2HnS6O2INt1xPxQPX5vjErRXFTZeTi0uF+dC0Num5BSzcc4EFuy+QkVdI57r+TOkeTtswX7v5OyptHYwSGCUwFuVsYhaj5+8n21DE3FGtaV/Hz9YmVUqkyaSlwt7tZrpjBXF3aqz250OlwnoVC1LfDFwXdz15e6Mzry507u4VwuVkryRl5TNvVzzf/HKRnAIjTzYIZGqPcJqH+tjatHtQAlMKlMBYh6vpeYyav59LKbl8Prw5TzcJtrVJdomUEpmbe/vDP72Ym+lu19Pd8YnfSoV1cbklDJqbyefOFUUJgeuSUmEV1udqeh5zd8SzbP8lCo0m+jatxpTudey6W4YSmFKgBMZ6pOcWMG7hAY5cTuf9gY0Z2a6mrU2yC4qSkkj+6isyN2zAmJb+4FRYB4c7Mpx0Pt53rii8Swhc33RROatEC3vnQnIOX247x8ojCUgJg1tWZ1K3cML83W1t2m+iWsUobIqPmxNLX27H1G8P887qEyRlGni9Zz278SGXN0WpqaTM+5q0b79FFhbi2asnTqE17gxcFxMKvbe3TVJhFdbnzPUsZm2LY93RqzjodYyIrMH4LrUJqVL5EmOUwCishquTnjkjW/HnVcf5YkscSdn5/GNgY7vN2bcGxvR0UhYsJHXJEqTBgHf/fvhPnoxTTbWie9w4lpDOjC1x/HgyEXcnPa90rs1LncPKpW2+rVACo7AqDnodHw1pSoCnMzO3niM5u4DpI1rg4mhfFceWxpiVReqixaQuXIgpOxuvPk/jP2UKznVU77bHjX3xKczYGsfO2GS8XBx47Ym6jO1YCx+3ypleXRyrCowQojfwOVpzz3lSyn/f9foY4D/AFfOuGVLKeebXPgb6og1F+wl4TUophRAbgWCz7TuBKVJKo/mcacBUoAj4Xkr5pjWfT1E6hBD88akIAjyc+fv6k4z8eh/zRrWplHNlTDk5pH6zlJT58zFlZODZ80n8p07DpX49W5umKEeklGw/m8TMrXEcuJCGv4cTf+odwYvtauDpUvne9/fDagIjhNADM4GeQAJwQAixVkp58q5DV0gpp951bgegI9DUvGsX0BXYBjwrpcwUmnM6ChgGLBdCdAcGAk2llPlCiKpWejRFGRnTMQx/T2deX/Erz875hUXjIgnyrhzuAVNeHmnfLiNl3jyMaWl4dO2K/6vTcG3UyNamKcoRk0ny48lEZm6N4/iVDIK9XXivf0Oea1MDV6fKvWovCWuuYCKBOCllPIAQYjmaANwtMCUhARfACRCAI5AIIKXMNB/jYH79ZhrcJODfUsp883E3LPMYCkvSr2k1qrg5MWHJIYZ8uYdF4yIJr+pha7PKjCk/n/QV/yP5q7kYk5Jx79iRgGlTcW3e3NamKcqRIqOJ9ceuMWtbHGcTs6np58a/BzdhcMuQx7oJrDWfvDpwudh2gnnf3QwRQhwTQkQJIUIBpJS/AFuBa+afTVLKUzdPEEJsAm4AWWirGIB6QGchxD4hxHYhRBuLP5HCInQM92f5+HbkFxkZOnsPhy+l2dqkh0YWFJC2fDnnnupN4gcf4BxWm5rfLKHG1/OUuDxG5BcZWbb/Ek98up3frfgVgM+HN+fn33dleGSNx1pcwLormJLyK+8uulkHLDO7tCYCi4AeQohwoAEQYj7uJyFEFynlDgAp5VNCCBdgKdADLUbjAFQB2gFtgP8JIWrLuwp9hBDjgfEANWrUsMBjKspC4+reRE/qwKj5+3n+q718+UIrukfYv1dTFhaSsWYNybO+pPDqVVxbtKDavz/EvV07W5umKEfyCowsP3CJuTviuZZhoGmIN3NGtqJng0C7b0BZnlhTYBKA0GLbIcDV4gdIKVOKbX4FfGT+fRCwV0qZDSCE2IAmHDuKnWsQQqxFc7v9ZL7fSrOg7BdCmAB/IOmue84F5oJWaPmIz6h4BGr6uRM1sQNjF+7n5cUH+WhIU4a2CvntE22ANBrJXL+epJmzKLx0CZfGjQn6+3u4d+qkalUeI7IMhSzZe5Gvd54nJaeAyDBfPhrSlM51/dX7oASsKTAHgLpCiDC0LLHhwPPFDxBCBEspr5k3BwA33WCXgFeEEB+irYS6Ap8JITwATynlNSGEA9AHLZMMYDXaamabEKIeWnwm2WpPp7AIAZ7OLB/fnolLDvHGd0dJyspnYtfadvOfVZpMZG3cSNKMmRTEx+McEUHIrJl4dO9uNzYqrE9aTgEL9lxg4e7zZBqK6FIvgKndw4kM87W1aXaN1QRGSlkkhJgKbEJLU54vpYwRQrwPHJRSrgVeFUIMQEsrTgXGmE+PQhOL42hutY1SynVCiEBgrRDC2XzNLcBs8znzgflCiBNAATD6bveYwj7xcHZg/pg2/OG7o3y08TQ3sgy807ehTV0NUkqyNm8mefoM8s+exSm8DtU/+wzPXj1VQ8fHiBtZBubtPM83ey+SW2DkqUaBTOkeTtMQ+2tAaY+oXmSqF5ndYDJJ/vH9SRbsvsCAZtX4ZFj5j2GWUpK9fTvJX0zHcPIkTrVq4T91Kl5P90boH78008eVhLRc5myPZ8XByxQZTQxoVo3J3cOpF+hpa9PsAtWLTGE1jiUdY925dQypN4QI3wiLXVenE7zbryFVPV34aONpUnMKmD2yFR7lMIZZSknOnj0kffEFhqPHcAwJIfjDD/Hu3091F36MiE/K5stt51h15ApCwJCWIUzsWodaFaABpT2iVjBqBfNQmKSJoeuGEpsWC0D74PaMaTyG9sHtLRqT+O7gZd5aeZyGwV4sGNsGfyuOYc7Zt5+k6V+Qd/AQDsHB+E+aiM+gQQjHx6fi+nHn1LVMZm6N44fj13As1oCymo+rrU2zS9QKRmEVNl3YRGxaLO+0e4fMgkyWnlrKhJ8mEOEbwZhGY+hVqxeOukf/YB7WOhQ/DycmLz3M0C/3sHhcW2r4WbbbbO7hIyR98QW5e/fiEBBA4Dt/xWfYMHSVdASv4l6OXEpj5tY4Np+6gYezA+O71OGlTmEEeKpxB5ZArWDUCqbUFJmKGLRmEA46B6IHRKMTOgqMBXwf/z0LYhZwPuM8we7BjGw4kiF1h+Dm+OiCcOhiGi8tOoCDTsfCsW1oXN37ka+Zd/w4SV9MJ2fnTvR+fvi98jJVhg9H51I52tYoHoyUkr3xqczcGseuuGR83BwZ2yGMMR1qVcr+eNbAKgPHhBA6wKNYu5YKjRKYh2NV7Cre3fMun3X/jCdqPHHHayZpYkfCDhacWMDhG4fxcvLiufrP8XyD5/F39X+k+8bdyGLU1/vJNBQxd2QrOoSX7XqGU6dImj6D7C1b0Pv44PfyS1R5/nl0bpVvDofiXqSUbDuTxIytcRy6mIa/hzPju4TxfNua5RLnswuKCuDcFjgRBU++B95lqzuzmMAIIb4FJgJG4BDgDXwqpfxPmSyzI5TAlJ5CYyH9VvWjiksVlvVd9sB4y9Gkoyw8sZCfL/2Mg86BAXUGMLrRaMK8w8p8/2sZeYyev58Lybl8+lwz+jWtVupz82NjSZoxk6xNm9B5euI3bixVRo5E71Fxe6ApSo/JJNkYc52ZW+OIuZpJdR9XJnStzbOtQyv92AgATEa4uBuOR8GptZCXBi4+MHQ+hD/x2+eXgCUF5lcpZXMhxAtAK+BPwCEpZdMHnlgBUAJTelacXsE/9/2T2U/OpmP1jqU652LmRRbHLGbNuTXkG/PpHtqdsY3H0qJqizLZkJFbyMuLD3DwYhrv9W/E6A61Hnh8/vnzJM+cReb336Nzc8N39Ch8x4xB72W/s84VlqPIaGLt0avM2naOuBvZhPm7M6lbHZ5pXr3y9wiTEq4c1lYqJ1ZC9nVwdIeIPtB4KNTpAQ5ljzVaUmBigObAt2jzWrYLIY5KKZuV2To7QQlM6TAUGei7si8hniEs7L3wobPFUvJSWH5mOctOLyMjP4NmAc0Y23gs3UO7oxMP9x/dUGhk2rIj/HQykandw/lDr3vHMBdcvkzyzFlkrF2LcHbG98UX8B03DocqVR7qXoqKSX6RkahDCczefo7LqXlEBHkypXs4fZoEo6/sfcIST5pFJRrSLoDeCer2gsZDoF5vcLKMO9iSWWRzgAvAUWCHEKImUCliMIrSseLMCm7k3eDfXf5dplRkP1c/pjSfwthGY1kdt5rFJxfzu62/o5ZXLUY1GsWAOgNw1pcua8fFUc+XL7TknTUnmLE1jqSsfP41SBvDXHj1KslfziZ91SqEXo/vyJH4vfIyDv6PFgNSVAxyC4r4dt8lvtoZT2JmPs1Cffhbv0Y80aBq5W7rk3peE5QT0XDjJAgdhHWFLn+EiH7garuuA2XKIhNCOEgpi6xgT7miVjC/TW5hLr2jexPhG8HcXnMtcs0iUxGbL25mQcwCTqacxM/Fj+cbPM9z9Z/D27l0WWJSSv7vp7N8sSWOgSGO/DF5L9nR2uQGn2HD8JswAcdA++/OrHh0Mg2FLPnlIl/vOk9qTgHtavsytXtdOob7VV5hyboOMau0uMoV82dYaFvN/dXoGfCw7nvfki6y14AFaLNX5gEtgLeklD9awlBbogTmt5l7bC7Tj0xnaZ+lNA2wbNhNSsmB6weYHzOf3Vd24+rgyuC6gxnZcCTVPUoaHXQnRcnJ7Hz/U6psXocDEs9nnqHa1Mk4Vit9AoCi4pKaU8D8XedZ9MsFsgxFdK8fwNQe4bSqWUkbUOamakH641FwYRcgIaiJ5v5qPAR8ym/8iCVdZOOklJ8LIZ4CAoCxaIJT4QVG8WAy8jNYeGIh3UK6WVxcAIQQRAZHEhkcydm0syyKWcSK0ytYfno5vWr1YmyjsTTwa3DPeUVpaaR+/TWpS78lKD+frC49mebcGhf/UBa5VUHJS+UmMdPAVzviWbrvEoYiI70bBTGle7hFaqTsjvxsOLNBi6vE/QymQvCtA13f1EQloL6tLXwgpVnBHJNSNhVCfA5sk1KuEkIckVKWLRXIjlArmAcz/ch05h6bS1T/KOr7ls8b+XrOdb45+Q1RsVHkFObQNrgt4xqNo3219piyskhZsIC0RYsx5eXh1bcv/lMm4xwWxp5zyUxYfAgPFwcWj4ukrmpKWOm4nJrL7O3n+O5gAkYpGdisGpO71yG8aiX7ty7Kh7jN2krl7EYozAXPatB4MDQZCsHNwcauP0u6yBagjToOA5qhtcnfJqVsZQlDbYkSmPuTakjl6ein6RzSmU+6flLu988qyOK7s9+x9ORSstITGXXCl267s9DnGPB86ikCpk7BuW7dO86JuZrBmAUHKCgyMX9M68rrKnnMiLuRzaxtcaz59Sp6IRjaOoSJXepYvHWQTTEZ4fwObaVyah0YMsDVV4unNB4KNdqDHY2JsKSL7CW0NOV4KWWuEMIPzU2mqMTMPz4fg9HA5OaTbXJ/TydPxtQeTt/dBSTNm4s+K4kDdQVbelWle48WDK1VjbvzzhpV82aleQzzC/P2MWNES55sGGgT+xWPTszVDGZtPccPJ67h7KBjdPtajO9SmyDvStLSR0q4vF8TlZjVkHMDnDyhQT/N/VW7G+grduuaUmWRmYeCdTFvbpdSrrOqVeWEWsGUzI3cG/RZ2Yenaj3Fvzr9q9zvbzIYSFu+nJSv5mFMScG9S2f8pk7hkG8GC04s4GDiQTwdPXm2/rO80OAFAtwC7jg/JTufsQsPEHM1kw8HN+HZ1qH3uZPCHjl0UWtAueX0DTydHRjVoSbjOobhZ8WO2uWGlJB4QnN/nVgJGZdA7wz1ntLcX3V7gaP9d3C2pIvs30AbYKl51wi0iZRvP7KVNkYJTMn8c+8/iT4bzdpBawn1LL8PZ1NBAen/+46UOXMoSkrCrX07Aqa9ilvLO8N9x5OOsyBmAT9f+hm90NO/Tn9GNxpNbe/at47JyS9i4jeH2BmbzB+fqs/kbnUqb8pqJUBKyZ5zKczYEscv8SlUcXNkXMcwRnWohbdrxf4WD0DKOa1O5XgUJJ8BoYc63TX3V0RfcKlY3SUsKTDHgOZSSpN5Ww8cUa1iKidXsq/Qb1U/BocP5p3275TLPWVhIekrV5E8ezZF167h2roVAa++intk5APPu5x5mUUnF7E6bjX5xny6hXS71YpGCEFBkYk3o46y+terjOlQi3f72XYMs+JepJRsOX2D6Vvi+PVyOlU9nRnfpTYjImvgXtEbUGZe1VYpJ6Lg6hFtX40O0GQINHwG3CtuAbCl58H4AKnm3ythLqDiJnOOzkGHjleavmL1e8miIjLWriN51iwKExJwadaU4H/+A/cOHUq12gj1CuWv7f7K5OaTWX5aa0UzeuNomgY0ZWwjrRXNp882x9/DmXm7zpOUnc+nzzbD2eExaHBo5xhNkg0nrjFz6zlOXdMaUP7zmcYMbRVSsRtQ5qTAqTVwPFprMImE4GbQ8x9aFlgZuxdbkryiPK7nXCfQLdAiIzUeRGkE5kNdgvnCAAAgAElEQVTgiBBiKyDQYjEV3j2muJcLGRdYe24tIyJGEOQeZLX7SKORzB82kDxzJgUXLuDSsCGBs7/Eo2vXMrmxfF18mdx8MmMbm1vRxCzm9W2vU9OrJqMajuKN3gOo6uXMBz+cJi2ngDkjW+HpUgncLhWQQqOJ1Ueu8OX2c8Qn5VA7wJ1PhjVjYPNqOOrtJ0vqocjPgtPfa+6v+K1gKgL/etDtbS1Y7x9ebqYUGAtIzE3kes51rudcv+P36znXuZ57nYz8DADmPDmHDtU7WNWe0gb5g9HiMALYB+iklFetalk5oFxkd/LmjjfZdnkbGwZvwM/Vz+LXlyYTWT/+RNKM6RTEncO5Xj38p03F88knLRofMZqMbL60mYUnFnIi5QS+Lr6MiBiBu6EL7625QESQJwvGtqGqZyXJRqoAGAqNfHcogdnbznElPY8GwV5M7R5O78ZBFbMBZaEBYn/U3F9nN0GRAbxDtVVK46Fahb2FY35FpiKS85LvEYzi2ymGlHvO83LyIsg9SPtxC7r1e9vgtlR1K1tLGasMHCt28UtSyvLrS2AllMDc5mzaWYauHcpLTV7itZav3f9AKeHXpdrQosgJUKPtb15bSkn2li0kTZ9B/unTONWuTcC0qXg+9RTCirn9UkoOJh5kwYkF7LyyE1cHV9r4PcWWfQ0IcA1m8bhIavm7W+3+Ci3Z4mYDyhtZ+bSs4cPUHuF0r18BG1Aai+D8Ns39dXo95GeCmz80GqRlgIVElrlWxSRNpBpS7xSPuwQkOS8ZozTecZ6bg9tt8SgmIIHugbe2reEGs7bAXJZSVvjcTyUwt3lty2vsv76fjUM23r/hpCET1v9Oy4bRO4GxAGp11rq2hnW55xublJKcnTtJ+mI6hhMncKxZg4ApU/Dq2xehL18/e1xaHAtjFvL9+e8xmkyQ0xSHrO4sfnFI5WwxYmMy8gpZtOcCC3afJy23kA51/JjaI5z2tStYA0qTCS7vu12rkpsMzl7QoL/m/grrCvoHRxqklGTkZ9yz2ii+fSP3BoWmwjvOc9I53SEegW6Bd4qJexCejp42+ftUK5hSoARGIyY5huHfD2dK8ylMbDax5IOuHIaocZB+Cbr/GdpOgMOLYfcX2jCjkEhNaOr2RAK5e/eS9PkX5P36K47VquE/ZTLeAwciHGybGZSYk8jSU0tZceZ/5BblIPPqMK3leMa3ebpiffDZKSnZ+Xy96zxLfrlIVn4RT0RUZUqPcFrWqECzeKSEa0fNc1VWQWYCOLhC/d6aqIT3BMfb7tWcwpw7hONazrVbApKYk0hibiJ5RXl33MJBOFDVreo9q43i4lHFuYrdvicfWWCEENOBkl4UwGgpZcVK3C4BJTAaE3+aSExKDBsGb8DD6a4xwlLC3i/hp3fBIxCGzIOa7W+/XmiAX7+BXZ9DxiVyjQ1IOulHbkw8DoGB+E+aiM/gwQinsk/PswZZBVksPL6ceccWY9KlE+gSxmutX6F3WG8cdSoB4GG5lpHH3B3xLNt/ifwiE32aBDOlWzgNq1Wgj4nkWHMBZDSkxILOAUPt7iTWe4LrQRFcL8i8Z+WRmJNIVmHWHZcRCAJcAx4oHn4ufuh1FTdbzhICM/pBJ0opF5XRNrtBCQwcSjzEmI1j+H2r3zO28V0dgHJTYfUkreFe/b4wcAa4ldzfK+/wQZI+fJec4+fRuxjxj3TDZ/wf0bV47jddCLYkOTuX55d9SYJpA3qXRALdAhnZcCRD6g65V2wV93ApJZcvt58j6tBlTBKeaV6dSd3qEF7V/v/uCk2F3Lh+jOsno7h+fgvXs69w3cGB655VSXT15Lopn7SCjHvO83XxvdddVUxAAtwCKv2XFKu6yCoLj7vASCkZu2ksFzMv8sPgH3B1KNai4uIeiHpJ8zn3+idEji8xKybvRAxJ078gZ/sO9FWq4PfyS1Rp5opu3xeQdAp8a0On30PT5x5pBrg1MRQaeXXZYX6+uIOwOgdILIzB09GTYfWH8UKDF8qcaVOZiU3MYta2c6w9ehW9TvBs6xAmdKlDqK99NKA0moxaxtVdcY/E3ESuZ17metZlkgtzkHe9pT0dPQjyCL5nxVE8eF7a6auVGSUwpeBxF5g9V/cw4acJvB35Ns83eF7baTLCzv/Ctg+hSi0YugCqNb/nXMOZMyRNn0725p/ReXvjN24cvi++gM7dnJVlMsGZH2DHf+Dar1oKZ8fXoMXIO/zX9oLRJHlnzQm+3XeJXs0L8AzaxeZLm9EJHf1q92NMozHU8aljazNtzokrGczYEsemk9dxcdDzYrsavNK5NlW9yu/fVEqpZVzd5aoq7r5Kyk2i6K6hu646R4JMEJSXRVBREUHOPgQFtyEovBdBQS0JcrdOxlVlRAlMKXicBUZKyQs/vEByXjLrB63HSe+kjWFd+YrWNrzJs9DvU3C+PWvDVFBA9patpEdHk7NzJzoPD3zHjMF39Cj0nveZySGlNihpx8daNo5HIHSYBq3GgrN9uVGklHz+cyyfbY6lR0RV/jygKivOfsPquNUYjAa6hnRlTKMxtApsZbfBV2tx4EIqM7bEsf1sEp4uDozpUIuxHcPwdbfsqlRKSaY51lFSkeBNMSkwFdxxnpPO6d54h4svQWkJBF7cT1D8bryKDAifGlqdSpOhULWhzeeqVFSUwJSCx1lgtl3exrQt0/h7h78zuO5giN0MqyZow436/Aeav3DrP5/hzFkyVkaTsWYtxvR0HIKC8BkyBN+RL6L38SndDaXUxrzu+A+c367Numg/WXO9udhXmvA3ey/y7poTNAv1Yf7oNqDPYfmZ5Sw7tYy0/DSa+DdhTKMxPFHjiQodqP0tpJTsiktmxpY49p1PxdfdiZc6hTGyfU28ytgJIbcw975Fgje378640gv9rYyre+o8zPt8XXw10TcWwrmtWgbY6e+hIBvcq94ugAxprUTFAliy2eUXJezOQOuovKaM9tkFj6vAmKSJYeuGYSgysLr/dzhu/RD2fAFVG8GwBRBQH2NWFpnff0969EoMx4+DoyOePXrgM3SI1ivsUepYLu+HHZ9A7CZw9oa246HtJHC3fPeAsrLxxDVeXf4roVVcWfxSW6r7uJJXlMfauLUsOrmIy1mXCfUMZXTD0QwMH4iLg/25/cqKySTZfCqRmVvjOJqQQaCXM+O71GFEZChuTvdP2Mg35t/jqrpbPLIK7s248nf1v3+th1sQ/q7+DxZykwku7dEywE6ugbxU7UtLgwHaSqVWZ6jEXwRsgSUFZi4QAXxn3jUEiAFC0YaQ/e4RbbUZj6vAbLywkT9u/yMftvwD/fYugSsHofU4ZM9/kvtrDOnRUWRt+hGZn49z3br4DB2C14ABOFSxcC3DtaOa0JxaC47u0GYctJ8GnvYxJGxvfAqvLDqIu7MDi8ZFUj9IcwMaTUa2XN7CghMLOJ58nCrOVRgRMYLhEcOp4lKB6j3uwmiSfH/8GjO3xHEmMYtQX1cmdQ1nSKvq6HQmknKT7rv6SMxNJNWQes81qzhXuSUcd686bu53LMtQLSm1DsUnorWOxVlXwdEN6j+trVTCnwAHFYy3FpYUmC1ALym1iJkQwgH4EegJHJdSNrSAvTbhcRSYIlMRg9YMwqEgl6j4WPRICjv+g4yTBtKjV1J4+TI6Dw+8+vXFZ8gQXBo3tn684cYp2Pmp5tbQOUKr0VpCgB10nj11LZPR8/djKDTy9Zg2tKl1O01bSsmhxEMsjFnI9oTtuOhdeCb8GUY1GlWuc3QeFUNhEUsPxLBg/xGu51wnoEoeTWpK3N2yuWGOgyQbkjFpEztu4eHocdtdVULWVaB74J2ZiZbgxmlzAWQ0pMZr75e6PbUCyPpPg5Nq/VMeWFJgzgCRUsoM87Y3sE9KGSGEOCKlbPGAc3sDnwN6YJ6U8t93vT4G+A9wxbxrhpRynvm1j4G+gA74CXhNSimFEBuBYLRO0DuBKVLebtAjhHjDfM0AKWXyg57tcRSYNWe+46973+f/riYRmRpGenI4OfsOg8mEW9u2+AwZjGfPnuhcbTBVL+Uc7Po/OLoMENB8BHR6XUt1tiGXU3MZPX8/V9LzmPF8S3qWMIb5XPo5FsYsZH38ekzSxJM1nmRs47E09m9sA4tvI6UkPT+9xJXHtezrnEtLILMwBcSdPa5c9C4PFI8g96DyqxNKu2heqURr0yCFTnN7NRmqtWxxrbirxoqKJQXmJeCvwDZut+v/AFgGvCel/ON9ztMDZ9FWOgnAAWCElPJksWPGAK2llFPvOrcDmkjcHNO8C3hbSrlNCOElpcwU2tfqKOA7KeVy83mhwDw0l14rJTB3UpgYwyuLhtPl10I6nXHGmFOAQ2Ag3oMH4TNoEE417KT7T/pl2P251orGVAhNhmm1NFUjbGZSak4BYxce4HhCOh8MasLwyJL/rm7k3mDpqaX878z/yC7Mpk1QG8Y0GkPn6p2tshLMKsgqUTwScxJvbecb8+84x0HngJvOj6xsDwryPQl0C+KJuvXoHBZOsLkGxNvZ27aZctk3IGaVFldJ2K/tC2mjub8aDbIbN+rjikWzyMzt+iPRBGZ/aVr1CyHaownQU+bttwGklB8WO2YMJQtMe2AG0Ml8zx3ASCnlqWLHOAIrgW+klCvM+6KAfwBrzNdVAgMYMzPJnPMe8Wu/xy1Jh9Tr8HqyJz5DBuPesWO5N54sNVnX4ZcZcGC+lt3WcAB0fgOCbTNMNbegiEnfHGb72ST+0LMeU3uE3/dDOLsgm+jYaJacXEJibiLhPuGMaTSGPmF9Sh1zuDkYqriA3Aqim7dzCnPuOEcndLfalNxdYe6u92NrTAHL96aRmWekc11/pnQPp22Yr32kXeelwal12krl/A6QJghsrLm/Gg/W6rIUdoGlBWYAt1cT26WU60pxzlCgt5TyZfP2SKBtcTExC8yHQBLaaud1KeVl82ufAC+jCcwMKeVfip23CU3wNqAJj9Fs4xNSyteEEBd4zAVGSknu/gNkfLeczE2bkIUmrgYIYtoHMelP3+HoZz8ZW79JTgrs+xL2zdFapNfrrQlNaJtyN6XQaOJPUcdYeeQKo9rX5G/9Gz1wnkmhsZCNFzayIGYBsWmxVHWryosNXmRg+MA7miQ+aDBUcfxc/O7boiTIXcu4ctDdmemVlJXPvF3xfPPLRXIKjDzZIJCpPcJpHlrKFHNrUpADZzZoohK3WevQXSVMc381HmrTVavi/ljSRfZvtGFjS827RqClKD9wqqUQYhjw1F0CEymlnFbsGD8gW0qZL4SYCDwrpewhhAhHi908Zz70J+BPUsodxc51Mds0G9gNbEVLRsh4kMAIIcYD4wFq1KjR6uLFiw98/opGYWIiGatWkb5yFYWXLqFzAq8auex5OpL33E8xv/cC2gSV/wezRchLhwNfwS+ztFTU2t20Ds41O5ZrbYPJJPlo42nm7IinT5Mg/u+55r85hllKye6ru1l4YiH7ru8r8RhvZ+97BKN42m6gW6BWEFtKrqbfbkBZaDTRt2k1pnSvQ0SQjRtQFhXAuZ8199eZDVCYA57B0GiwNq++WktVq2LnWFJgjgHNpdRSSMyxlSNSygf6KUrjIrvreD2QKqX0FkL8EXCRUv7D/Nq7gEFK+fFd54xGE785wM9ArvmlEOAqmqBdv5+NlWUFIwsKyNq6jfToKHJ27dYC9hEh+PiewrO+J4ahM3j64PvUrVKXeb3m2drcRyc/Gw4t0EYF5NyAGu2hyxtQ54ly/WCatzOef35/ina1fZk7qnWpiw9jUmL45eovd6xGLDkf/UJyDl9uO8fKIwlICYNbVmdSt3DCbDlczWTUCm1PRMHJtWBI14LzDQdqK5WaHVStSgWitAJT2ja3PsDNJPfSll0fAOoKIcLQssSGA8/fZWSwlPKaeXMAcDPGcgl4RQjxIZqLrCvwmRDCA/CUUl4zp0v3AXZKKY8DVYtd9wKlcJFVdAxnz5IRvZKMtWsxpqXhEBiI39iR+LgfxOnGT5oraeAsvo1fRaohlWktpv32RSsCzh5au5k2L8ORb2DXZ/DNEKjWQlvR1Hu6zJMFH4aXO9fG38OZN747ynNz9rJobJtS9eRq5NeIRn6NLG7PmetZzNwax/pjV3HQ6xgRWYPxXWoTUsVG/bWkhCuHtJVKzCptbpCjO0T01VxgtbvbbQNUhWUojcB8CBwRQmzldhbZA91jAFLKIiHEVGATWpryfClljBDifTQX21rgVXPspAhNwMaYT48CegDH0WbSbJRSrhNCBAJrhRDO5mtuQXORPTZoFfY/kL5yJYZjx25X2A8ZjHuoA2LVK5CcCE99AO0mk1mYxfwT8+ka0pVmAc1sbb5lcXSFyFeg5Wg4tlyrpVn+vNZjqvMftGwjK38rfqZFdaq4OzHpm0MM/nIPS15qW+4rhaOX05mxNY6fTibi7qTnlc61ealzGFU9bdRdIDHm9lyV9Iva9NO6vTRRqfsUOKmGko8LD5NF1gZNYPYButJkktk7FcVFJqUk98ABMqKjydz0I9JguF1h378/Dj4+sOtT2PoB+IRqHZCrtwRgxpEZzDk2h//1+x8N/BrY+EmsjLEIYlZq3QGSz4BfuHlUwLNQlmrxh+Do5XTGLjyAABaMbUPTEOsH0PfFpzBjaxw7Y5PxcnFgbMcwxnashY+bDVYFqec199fxaG1Mg9BD7a6a+6tBP7vrN6d4NNTI5FJg7wKjBexXk75ypRaw9/DAq29ffIYWq7DPSoRV4yF+m5bO2e8zcNGCuGmGNHpH96Zj9Y582u1T2z5MeWIywel1WmPN68fBuwZ0+h20eNGq7UPik7IZNX8/qTkFzH6xFV3qBVj8HlJKtp9NYubWOA5cSMPfw4mXOtXmxXY18CxjA8oyk3lNc32diNJcYQCh7bSVSsNnwMPyz6+wD6wtMJellBWnF8Z9sEeBuRWwXxlNzs5dWsA+MlKrsO/V684K+3NbYOV4Lej99EfQctQdQe5PD37KopOLWDVgFbV9bFsNbxOkhNgfYfvHWr81z2Do8Cq0GmM1N82NTAOjFxwgNjGL/z7bjIHNq1vkuiaT5MeTWgPK41cyCPZ2YUKX2jzXpgauTuUYHM9N1XrHHY/SgvZICGqirVQaDwafCv+9U1EK1AqmFNiTwOTHxpIeFX1HwN570DNahX3NmncebCzU3GG7/g8CIrQOyFXvdH8l5SbRZ2UfetbsyQedPyjHJ7FDpNRGBOz4BC7sBDd/aD9FSxJwsXzKbqahkPGLD7I3PpW/9m3Ay53LLu5FRhPrj11j1rY4ziZmU9PPjUld6zC4ZQhODtZPZAC0LzBnftBE5dzPYCrS3I+Nh2qr5oB65WOHwm545CwyIcR0tAD7PS+hZZUpHhFjVhaZP2wgPTr6dsC+e3etJf79KuzTL0H0y9rwrpajofe/S/w2/tXxrygyFTGp2aRyeBI7RwitZqZ2N7i0VxOan/8Ouz/TxgS0nQBuvg++xkPg5eLIwrGR/P5/v/LP70+RlJ3PW70jHqpaPr/IyMrDV5i9/RwXU3KpF+jB58Ob07dJMA76chCWonyI/Ulzf53ZCEV54FUd2k3ShCW4mapVUfwmD8oie9BXe/v42l8BuR2wX6lV2BsMONcNp+pbf8J7wAAcfB/wQXdqPayZrMUYhs7Xvj2WwNXsq3x39jueqfsMoV4V3pNpWWq0gxejtFbvOz6B7f/W2tG0eRnaT7VY3MDFUc/0ES3xc49hzvZ4krLy+WhIUxx/QxzyCowsP3CJuTviuZZhoGmIN3NGtqJng0B0D+gYYBGMRXBhhxaoP7UO8jPAzQ+aP6/FVULblUv6t6LycF+BkVIuunufECLoQYWLivtzK2C/aiWFF7WAvffAgfgMGYxLkyYP/nZbaICf3oH9c7Vaj6HzH9hheM6xOQgEE5pOsMKTVBKqtYDhS7WU2p3/1Qau7ZujxWc6TAPvR4+d6HWC9wc2IsDTmU9/OktqTgGzXmhZ4tCuLEMhS/Ze5Oud50nJKSAyzJePhjSlc11/6/YJk1IbAHfCXKuSkwROnlrmV+OhWiaYlTPwFJWXh4rBCCEOSylbWtGecsXaMRhZUEDWtm3mGfbmgH2bNvgMHXJvwP5+JMdB1BgtG6rdFHjyvQcWp13MvMjA1QMZHjGctyLfstSjVH6S47SY1rHlWjv45i9omWcWarC4bP8l/rLqOE1DfJg/ps2tWfZpOQUs2HOBhbvPk2kooku9AKZ2DycyzHIuu3uQUns/3RzWlXEJ9M5Q7ylzrUovrcZIobgPVgny/9b8l4qGtQQmPzaW9JsV9qmpOFStivegQfgMLiFg/yCOLof1v9dSa5/5Eur3/s1T3tr5FlsubeGHwT/g7+r/CE/xmJJ2URsVcGSJ1t6k6XPQ+ffgX/eRL70p5jqvLjtC9Squ/HdYMzacuM43ey+SW2DkqUaBTOkebt36mZRz5gLIKEg+q9Wq1OmhiUr9PlZJeFBUTqwlMJOllLMeyTI7wpICY8zONlfYR2M4WixgP2Qw7p06PVxL/Pxs+OGPcPRbrZHj4K9K5bKJTYtlyNohjG08ltdbvf4IT6Mg8yrsmQEH50ORQesK0PkPEPRoA8T2n0/l5UUHyDQUoRMwoFk1JncPp16gp4UMv4uMK1rx6fEouPartq9mRy1+1/AZcK9AXbUVdoNV05QrC48qMFJK8g4eJD0q+o6AvfeQIb8dsL8f14/Dd2MhJQ66/gm6vlnqdievb32dvdf2snHIRrydVeW0RchOgr2zYP9XUJClfdPv8gZUb1XmS55NzGLNr1cY1iqUWtZoK5OTAidXay6wi3sACcHNtZVKo8EWiS8pHm+UwJSCsgpMYWIiGavXkL4yWgvYu7vfrrD/rYD9/ZASDsyDTX/RuswOmQdhnUt9ekxKDMPXD2dys8lMaq5Sky1OXhrsm6uJjSFd69zc5Q2tC7A9kJ8Fp7/XVirxW7VaFf96t2tV/MNtbaGiEqEEphSUVWCuvv1nMlateviA/f3IS4e107QK6fCeMGg2uD9c/GTS5kkcTz7OxsEby29W+uNIfhYc+FpLbc5J0txNXd7QOgOXd11IoQFiN2miEvuj5srzrqFV1DceolXYq1oVhRVQAlMKyiowBZcugZQPF7C/H5cPQNQ4yLoKT/xNq8V4yFqDIzeOMGrDKF5v9TrjGo97dJsUv01BLhxerCUEZF3VXGZd/qiNSLDmh7qxEOK3a4H6U+s1t517gBYjajxUm1uvalUUVsbS82AUxXCqYYEuOSaTVnux5R/gVQ3GbYKQ3/z3KpEZR2bg5+LH8PrDH90uRelwcoN2E6H1WPj1W62b9bLhENgEuvwBGgyw3KgAkwku79VWKidXQ24KOHtrw7qaDIFaXUCv/isr7A/1rrQF2UmwaoLW16nhM9D/c3AtW3rq3mt72X99P29FvmWxiYiKh8DBWROZFi9qArDzv/DdGC3+0fkP2qqiLB/+UmpZXyei4cQqyEwAB1eo/7Tm/qrb06qdoRUKS6BcZOXd7DJ+O6x8BQwZ0PtDaDW2zC4VKSUvbniRG7k3+H7Q9w81r11hJUxGOLlGE5rEE1qhZqfXodmI0glC0lnN/XUiWssk1DlA+JOaUNV/WpvmqVDYGOUiszeMRVrfqx2faEV7I1dB4KONzd15ZSfHko7xt/Z/U+JiL+j0WpC94TNwdqM2k2bda9rIgI6vaSMV7q6ST79sXqlEaWnqCKjVSWtZ02CARRtxKhTliVrBlMcKJiNB64B86RfNlfL0x+D0aPUPJmniufXPkVOYw5pn1uCoU/2i7BIptbk9Oz6BS3vAvSp0mKoJR9xmza12ea92bPVW2kql0SDwCrat3QrFA1ArGHvh9A9aB2RjIQyeB02HWeSymy9u5nTqaT7o9IESF3tGCAh/Qvu5sBt2fgI/vav9AAQ0gB5/1eIqD2hgqlBURJTAWIuifPjpb7DvS212xtAF4FfHIpc2mozM/HUmtb1r0yesj0WuqSgHanXUfhIOwcXdmug8optUobBnlMBYg5RzEDUWrh2FthOh5/sWzfj54fwPxGfE89+u/0VvqVRYRfkR0kr7USgqOUpgLM3xKFj3Oy3YO3wZRFh2hVFoKmTWr7No4NuAJ2s+adFrKxTlTWFhIQkJCRgMBluboigBFxcXQkJCcHQsmxteCYylKMiBDW/CkW+0yX9DvwbvEIvfZnXcahKyE5j5xEx0QlVsKyo2CQkJeHp6UqtWLesOVlM8NFJKUlJSSEhIICwsrEzXUAJjCRJPasV1yWeh8xvQ7W2rVFbnG/OZc3QOTQOa0rl66RthKhT2isFgUOJipwgh8PPzIykpqczXUALzKEgJhxbCxrfA2UurbanT3Wq3izobRWJuIv/q9C/1H1JRaVDvZfvlUf9tlMCUFUOGVkAXs0qbCjhoDnhUtdrtcgtzmXtsLpFBkbQNbmu1+ygUCoWlUE78spBwCGZ3hpNr4cn34IVoq4oLwLLTy0g1pDKtxTSr3kehUFiPbdu20a9fv0c+5kFIKXn11VcJDw+nadOmHD58uMTj/vKXvxAaGoqHh/XaDymBKQtHl2nusXEbtT5TVm6PnlWQxfwT8+lcvTPNqza36r0UiscZKSUmk8nWZjwSGzZsIDY2ltjYWObOncukSSUPIOzfvz/79++3qi1KYMpCr3/AxB0QGlkut1tycgmZBZlMbTG1XO6nUDxOXLhwgQYNGjB58mRatmzJkiVLaN++PS1btmTYsGFkZ2cD8MMPPxAREUGnTp149dVXH7jK2L9/Px06dKBFixZ06NCBM2fO3HPMe++9x8iRI+nRowd169blq6++uvVadnY2Q4cOJSIighdeeIGbLb3ef/992rRpQ+PGjRk/fjwltfpas2YNo0aNQghBu3btSE9P59q1a/cc165dO4KDrduSSMVgyoKj670NC61EuiGdxScX0zXe8SUAABmbSURBVLNmTxr6NSyXeyoUtuDv62I4eTXTotdsWM2Lv/X/7W4JZ86cYcGCBbz//vsMHjyYzZs34+7uzkcffcSnn37Km2++yYQJE9ixYwdhYWGMGDHigdeLiIhgx44dODg4sHnzZv785z8THR19z3HHjh1j79695OTk0KJFC/r27QvAkSNHiImJoVq1anTs2JHdu3fTqVMnpk6dyrvvam2GRo4cyfr16+nfvz+zZ88GYOLEiVy5coXQ0NBb9wgJCeHKlStWF5OSUAJj5yyIWUBuYS6Tm022tSkKRaWlZs2atGvXjvXr13Py5Ek6duwIQEFBAe3bt+f06dPUrl37Vj3IiBEjmDt37n2vl5GRwejRo4mNjUUIQWFhYYnHDRw4EFdXV1xdXenevTv79+/Hx8eHyMhIQkK0OrrmzZtz4cIFOnXqxNatW/n444/Jzc0lNTWVRo0a0b9/fyZOnHjrmiWtamyVqacExo5Jzkvm21Pf0qd2H8KrhNvaHIXCqpRmpWEt3N217uZSSnr27MmyZcvueP3IkSMPdb133nmH7t27s2rVKi5cuEC3bt1KPO7uD/6b287Ot1tL6fV6ioqKMBgMTJ48mYMHDxIaGsp7771XYgeEkJAQLl++fGs7ISGBatWqPZT9lkLFYOyYecfnUWgqVKsXhaKcaNeuHbt37yYuLg6A3Nxczp49S0REBPHx8Vy4cAGAFStWPPA6GRkZVK9eHYCFCxfe97g1a9ZgMBhISUlh27ZttGnT5r7H3hQTf39/srOziYqKKvG4AQMGsHjxYqSU7N27F29vb5u4x8DKAiOE6C2EOCOEiBNCvFXC62OEEElCiF/NPy8Xe+1jIUSMEOKUEOILYZb2/2/v3qOrqK8Fjn83EEhoBEQeRSIFFBCBNEDEQFio2AcPRbDq1XoVaAVfWMHbm0otyorWFy5KNVAeFyk+irX0ioAgRcFaKCgUQoAk8iqPCFEIFxCJRMK+f8yceAgnz5OZCbA/a2XlnJnfzNn5ZXJ2fjNzfltE3hORTe666SJS110+SURyRSRLRN4WkerVIK4l8r/K561P32LoFUNp06hN0OEYc0Fo3rw5f/zjH7nzzjtJTEwkJSWF3Nxc4uLimDZtGgMGDKBv3760bNmSxo0bl7mftLQ0xo8fT2pqKsXFxWW269WrF4MHDyYlJYUJEyaUO9Jo0qQJo0aNolu3bgwdOvSMZDR9+vSS6zCDBg2iffv2XHHFFYwaNYpp06aVtEtK+vYu1LS0NBISEjhx4gQJCQlMnDixMl1UNarqyRdQF9gJtAfqA5uAq0q1GQFkRNi2D7Da3UddYA1wnbuukftdgL8Cd7jPfwTUcx8/DzxfUYw9e/bU2urJ1U9q91e76/4v9wcdijGeyc7ODjqESvvyyy9VVfX06dP6wAMP6OTJk6Pa35NPPqmTJk2qidA8Fel3BKzXSuQBL0cwvYAdqrpLVYuAN4GbK7mtArE4iakBEAN8DqCqodtM6rnr1V3+N1U95a5bC9T8TJM+2XtsLwt2LOC2jrfRKt4qGxpTG8yaNYukpCS6dOnC0aNHue+++4IOqdbz8iJ/a2Bf2PM8INIcJz8RkX7ANmCcqu5T1TUishI4gDNSyVDVnNAGIrIMJ4EtBSKdiPwZUP5J0lps+qbpxNSJ4d5u91bc2Bjji3HjxjFu3Lgzls2ZM4ff//73ZyxLTU1l6tSpFe7Pk1NStYyXCSbSfXGl759bBMxT1ZMicj8wF+gvIlcAnfl2FLJcRPqp6kcAqvpjEYkF3gD6A8tLXlTkceCUu+7soERGA6MB2rSpfdc2dh7ZyeJdixnRZQTNGzYPOhxjTDlGjhzJyJEjgw6j1vLyFFkecFnY8wRgf3gDVS1Q1ZPu01lAqMzfMGCtqh5X1eM4I5WUUtt+DSwk7LSbiAwHbgTucs8TnkVVZ6pqsqomN29e+97Ap2ZOpWFMQ37W9WdBh2KMMVHxMsGsAzqISDsRqQ/cgZMQSohI+AWGIUDoNNhe4FoRqSciMcC1QI6IxIe2EZF6wCAg130+APgVMERVT3j4c3kmpyCH5XuWc/dVd9Mk9py+Cc4YY7w7Raaqp0RkDLAM506wV1R1q4ik49yBsBD4hYgMwTmldRjnrjJwrqv0BzbjnFZ7T1UXiUhLYKGINHD3uQKY7m6TgXNDwHL3jua1qvrtx1vPARmZGTSq34h7rron6FCMMSZqnn6SX1WXAEtKLXsi7PF4YHyE7YqBs27RUNXPgYifRFLVc/qj7plfZPJR3kc80uMRLqp/UdDhGGNM1OyT/LVExsYMmsY25adX/jToUIwxHvGjHkxubi69e/emQYMGvPjii9XeT02wuchqgY8PfMzH+R+TdnUaDWMaBh2OMReskg8IelzjyUtNmzblpZdeYsGCBUGHYiOYoKkqGRszaNGwBbd3uj3ocIy54Jxv9WBatGjB1VdfTUxMTLRdEzUbwQTsH5/9g8yDmUxImUCDug0q3sCY89XSxyB/c83u87vdYOBzFTY7n+rB1CaWYAIUGr0kxCcwrMOwoMMx5oJ1PtWDqU0swQTog70fkHM4h9/2/S0xdYIfzhoTqEqMNLxyPtWDqU3sGkxAik8Xk7Exg3aN2zG43eCgwzHGcH7Ug6lNbAQTkKW7l7Lz6E5evPZF6tapG3Q4xhjOrAdz8qQzi9XTTz9Nx44dS+rBNGvWjF69epW7n7S0NIYPH87kyZPp379/me1C9WD27t1bUg9m27ZtEduG14Np27btWfVgwLkGk5+fT3JyMseOHaNOnTpMmTKF7OxsGjVqVNXuiJqUMWXXBSE5OVnXr1/v++t+c/obhi4YSly9ON666S3qiA0kzYUpJyeHzp07Bx1GpRw/fpz4+HhUlYceeogOHTqcNbtyVUycOJH4+Hh++ctf1mCUNS/S70hE/qWqyRVta+9sAVi4YyF7v9zLmO5jLLkYc46wejBVZyMYn0cwRcVFDH57MC3iWvD6oNfPushnzIXkXBrBRBJNPZhzRTQjGLsG47P52+aT/1U+6X3SLbkYc46zejDls/MzPio8VciszbNIbplMSquUijcwxphzmCUYH72Z+yaHCg/xcPeHbfRijDnvWYLxyfGi47yy5RVSW6fSo2WPoMMxxhjPWYLxyWs5r3Hk5BEeTno46FCMMcYXlmB8cPTkUV7d+io3tLmBLs26BB2OMSYgtakezIgRI2jXrh1JSUkkJSWRmZlZ7dcsi91F5oM5W+bw1Tdf8WDSg0GHYowpx4VWD2bSpEnceuutnsVy7vbiOeJQ4SH+lPsnBrQbQMeLOwYdjjGmFKsH4x0bwXhs9ubZFBUX8eD3bfRiTHme/+R5cg/n1ug+r2x6Jb/q9asK212o9WAef/xx0tPTueGGG3juuefOmMW5JtgIxkP5X+Xz1qdvMeTyIbRt3DbocIwxZQjVg1m7dm1JPZikpCTmzp3Lnj17ItaDKc/Ro0e57bbb6Nq1K+PGjWPr1q0R24XqwTRr1qykHgxQUg+mTp06JfVgAFauXMk111xDt27dWLFiRcl+77///ionl2effZbc3FzWrVvH4cOHef7556u0fWXYCMZDM7NmcprT3P/92lkMyJjapDIjDa9ciPVgWrVqVfJaI0eOLPeGgOqyEYxH9n25j7e3v82tHW7l0vhLgw7HGFMJF1I9mAMHDgBOUl2wYAFdu3aNan+RWILxyPRN06lbpy6jEkcFHYoxppLC68EkJiaSkpJCbm4ucXFxJfVg+vbtS8uWLWncuHGZ+0lLS2P8+PGkpqZSXFxcZrtQPZiUlJSSejBlCa8HM3To0LPqwYSuw+Tn55OQkMDkyZN5+umnSUhI4NixYwAMGjSI/fv3A3DXXXfRrVs3unXrxqFDh/jNb35Tpb6qDJtN2YPZlHcd2cWwhcO456p7+K/k/6rx/RtzvjiXZlO2ejDfsnowAZq2aRqxdWMZ2dVmWTXmfGH1YKrOLvLXsNzDuSzbvYzRiaNpGts06HCMMTVk3LhxZ41YoqkHM3HixJoMr1ayBFPDpm6cykX1L2J4l+FBh2KM8ZjVgymfnSKrQVkHs/gw70NGdhlJo/qNgg7HGGMCZQmmBr288WWaxjblrs53BR2KMcYEzhJMDVmXv461B9by864/p2FMw6DDMcaYwFmCqQGqSsbGDFrEteD2TrcHHY4xxtQKlmBqwOr9q9nwxQZGJ44mtl5s0OEYY2opP+rBvPHGGyQmJpKYmEifPn3YtGlTtfcVLU8TjIgMEJFPRWSHiDwWYf0IETkoIpnu171h614Qka0ikiMiL4k7SY+IvCcim9x100Wkrru8qYgsF5Ht7veLvfzZQlSVlze+TOv41tzS4RY/XtIY4xFV5fTp00GHEZV27drx97//naysLCZMmMDo0aMDi8Wz25TdN/6pwA+BPGCdiCxU1exSTf+sqmNKbdsHSAUS3UWrgGuBD4HbVfWYm3DmA7cBbwKPAR+o6nNuMnsM8Hz2vBX7VpBdkM1TqU8RUzf4+gvGnKvyn3mGkzk1O11/g85X8t1f/7rcNrt372bgwIFcf/31rFmzhrFjxzJ9+nROnjzJ5Zdfzpw5c4iPj2fJkiU8+uijNGvWjB49erBr1y4WL14ccZ+ffPIJY8eOpbCwkLi4OObMmUOnTp3OaDNx4kR27tzJZ599xr59+0hLS2PUKGdqqVA9mC1bttCzZ09ef/11RIT09HQWLVpEYWEhffr0YcaMGWdNmNmnT5+SxykpKeTl5VWn62qElyOYXsAOVd2lqkU4SeDmSm6rQCxQH2gAxACfA6jqMbdNPXd9aK6bm4G57uO5wNBof4CKFJ8uJmNjBm0bteXG9tUf0hpjgvXpp59yzz33sHz5cmbPns3777/Phg0bSE5OZvLkyXz99dfcd999LF26lFWrVnHw4MFy9xeqB7Nx40bS09P5dRlJLisri3fffZc1a9aQnp5eMk/Yxo0bmTJlCtnZ2ezatYvVq1cDMGbMGNatW8eWLVsoLCwsSXDhc5GFmz17NgMHDoyma6Li5QctWwP7wp7nAddEaPcTEekHbAPGqeo+VV0jIiuBA4AAGaqaE9pARJbhJLClOKMYgJaqegBAVQ+ISIsa/4lKWbZ7GTuO7GBSv0nUq2OfWTUmGhWNNLwUqgezePHiknowAEVFRfTu3TtiPZiZM2eWub+jR48yfPhwtm/fjojwzTffRGwXqgcTFxdXUg+mSZMmJfVggJJ6MH379mXlypW88MILnDhxgsOHD9OlSxduuummiLVgVq5cyezZs1m1alW03VNtXo5gJMKy0jNrLgLaqmoi8D7uCERErgA6Awk4iaq/m4Scnaj+GGiFM7rpX6WgREaLyHoRWV/RfyHlOXX6FNM2TaPjxR35UdsfVXs/xpjgla4Hk5mZSWZmJtnZ2cyePTtiaeLyhOrBbNmyhUWLFpVZt6U69WDmz5/P5s2bGTVqVJn7zcrK4t577+Wdd97hkksuqVLsNcnLBJMHXBb2PAHYH95AVQtU9aT7dBbQ0308DFirqsdV9TjOSCWl1LZfAwv59rTb5yLSCsD9/kWkoFR1pqomq2py8+bNq/3DLdq5iD3H9jAmaQx1xG7GM+Z8cD7Ug9m7dy+33HILr732Gh07diw3Tq95+c64DuggIu1EpD5wB05CKBFKCK4hQOg02F7gWhGpJyIxOBf4c0QkPiyJ1AMGAaGrgguB0ARgw4F3PPiZACgqLuIPm/5A10u6ct1l13n1MsYYn50P9WDS09MpKCjgwQcfJCkpieTkCmfV94yn9WBEZBAwBagLvKKqvxWRdGC9qi4UkWdxEssp4DDwgKrmunegTQP64ZxWe09VHxWRlsBinFNjdYEVONdtTonIJcBbQBucBHWbqh4uL77q1oOZlzuPZz5+hhk/mEGf1n0q3sAYE5HVgzm/68F4emVaVZcAS0oteyLs8XhgfITtioGzii2o6udAxDGkqhYAN0QZcqWcOn2Kfgn96H1pbz9ezhhTC8yaNYu5c+dSVFRE9+7drR5MJVhFy2pWtFTVsy7QGWOq5lwawUQSTT2Yc0WtHcGczyy5GGOsHkz57PYnY0ygLuSzKLVdtL8bSzDGmMDExsZSUFBgSaYWUlUKCgqIja3+BL52iswYE5iEhATy8vIqnHrFBCM2NrZkRoHqsARjjAlMTExMyfQr5vxjp8iMMcZ4whKMMcYYT1iCMcYY44kL+oOWInIQ2BN0HGVoBhwKOohyWHzRsfiiY/FFL5oYv6eqFc4WfEEnmNpMRNZX5pOyQbH4omPxRcfii54fMdopMmOMMZ6wBGOMMcYTlmBqr7LrsdYOFl90LL7oWHzR8zxGuwZjjDHGEzaCMcYY4wlLMAEQkQEi8qmI7BCRxyKsf1REskUkS0Q+EJHvha0rFpFM92th6W19im+EiBwMi+PesHXDRWS7+zW89LY+xfe7sNi2iciRsHWe9p+IvCIiX4jIljLWi4i85MaeJSI9wtb50XcVxXeXG1eWiPxTRL4ftm63iGx2+656hZSij+86ETka9jt8ImxduceFT/H9d1hsW9zjram7zo/+u0xEVopIjohsFZFHIrTx7xhUVfvy8Qun1PNOoD1QH9gEXFWqzfVAQ/fxA8Cfw9YdrwXxjQAyImzbFNjlfr/YfXyx3/GVav8wTrluv/qvH9AD2FLG+kHAUkCAFOBjv/qukvH1Cb0uMDAUn/t8N9As4P67Dlgc7XHhVXyl2t4ErPC5/1oBPdzHFwHbIvz9+nYM2gjGf72AHaq6S1WLgDeBm8MbqOpKVT3hPl0LVH86Uw/iK8ePgeWqelhV/w9YDgwIOL47gXk1HEOZVPUj4HA5TW4GXlXHWqCJiLTCn76rMD5V/af7+uD/sVeZ/itLNMdtpVUxPl+PPQBVPaCqG9zHXwI5QOtSzXw7Bi3B+K81sC/seR5nHwDhfo7z30ZIrIisF5G1IjI0wPh+4g6v54vIZVXc1o/4cE8ttgNWhC32uv8qUlb8fvRdVZU+9hT4m4j8S0RGBxQTQG8R2SQiS0Wki7usVvWfiDTEeXP+a9hiX/tPRNoC3YGPS63y7Ri06fr9F6nWcsRb+UTkP4Fk4NqwxW1Udb+ItAdWiMhmVd3pc3yLgHmqelJE7gfmAv0rua0f8YXcAcxX1eKwZV73X0XKit+Pvqs0EbkeJ8H0DVuc6vZdC2C5iOS6/9H7aQPONCXHRWQQsADoQC3rP5zTY6tVNXy041v/iUg8TnIbq6rHSq+OsIknx6CNYPyXB1wW9jwB2F+6kYj8AHgcGKKqJ0PLVXW/+30X8CHOfyi+xqeqBWExzQJ6VnZbP+ILcwelTlH40H8VKSt+P/quUkQkEfgf4GZVLQgtD+u7L4C3cU5L+UpVj6nqcffxEiBGRJpRi/rPVd6x52n/iUgMTnJ5Q1X/N0IT/45BLy842VfEi3D1cC6etePbi5FdSrXpjnPBskOp5RcDDdzHzYDt1PCFzErG1yrs8TBgrfu4KfBvN86L3cdN/Y7PbdcJ56Kq+Nl/7r7bUvZF6sGceYH1E7/6rpLxtQF2AH1KLf8OcFHY438CAwKI77uh3ynOG/Rety8rdVx4HZ+7vjHOdZrv+N1/bl+8Ckwpp41vx6CdIvOZqp4SkTHAMpw7X15R1a0ikg6sV9WFwCQgHviLiADsVdUhQGdghoicxhl9Pqeq2QHE9wsRGQKcwvlDGuFue1hEngLWubtL1zNPEfgVHzgXWN9U9y/H5Xn/icg8nDudmolIHvAkEOPGPh1YgnMXzw7gBDDSXed531UyvieAS4Bp7rF3Sp0JEVsCb7vL6gF/UtX3AojvVuABETkFFAJ3uL/jiMdFAPGB80/X31T1q7BNfek/IBW4G9gsIpnusl/j/OPg+zFon+Q3xhjjCbsGY4wxxhOWYIwxxnjCEowxxhhPWIIxxhjjCUswxhhjPGEJxhhjjCcswRhzDnCnqV8cbRtj/GQJxpga4NbYsL8nY8LYH4Qx1SQibd3CTtNwJmG8W0TWiMgGEfmLO+EgIjJIRHJFZJVb6KnMUYaI9BKn0NdG93unCG0mishrIrLCLQw1Kmx1vDvDda6IvCHuR8dF5AkRWecWwZoZWm6MlyzBGBOdTjhzP/0QZ/bhH6hqD2A98KiIxAIzgIGq2hdoXsH+coF+qtodZ9qWZ8pol4gzp1Rv4AkRudRd3h0YC1yFU3wr1V2eoapXq2pXIA64sco/qTFVZHORGROdPaq6VkRuxHlTX+0ODuoDa4ArgV2q+m+3/TygvFogjYG5ItIBZ6r0mDLavaOqhUChiKzEmfjxCM7EhXkA7lxUbYFVwPUikgY0xJnUcCtO2QVjPGMJxpjohCY0FJxqgHeGrxSRqpYDeApYqarD3IJRH5bRrvQkgqHnJ8OWFQP13FHUNCBZVfeJyEQgtopxGVNldorMmJqxFkgVkSvAqWgoIh1xTnm1d5MFwH9UsJ/GwGfu4xHltLtZRGJF5BKc2X3XldM2lEwOudeFbq0gBmNqhCUYY2qAqh7ESQjzRCQLJ+Fc6Z7GehB4T0RWAZ8DR8vZ1QvAsyKyGmfa+bJ8Arzrvs5T6hazKiO2IziF4TbjVIAsLxkZU2Nsun5jPCYi8eqU+BVgKrBdVX8Xxf4mAsdV9cWaitEYL9gIxhjvjXIvuG/FOQU2I+B4jPGFjWCMCYCIjAQeKbV4tao+FEQ8xnjBEowxxhhP2CkyY4wxnrAEY4wxxhOWYIwxxnjCEowxxhhPWIIxxhjjif8HqMUYmSDyzFAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb84b470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch5_1.best_score_, gsearch5_1.best_params_))\n",
    "test_means = gsearch5_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch5_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch5_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch5_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch5_1.cv_results_).to_csv('my_preds_reg_alpha_reg_lambda_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "train_scores = np.array(train_means).reshape(len(reg_alpha), len(reg_lambda))\n",
    "\n",
    "#log_reg_alpha = [0,0,0,0]\n",
    "#for index in range(len(reg_alpha)):\n",
    "#   log_reg_alpha[index] = math.log10(reg_alpha[index])\n",
    "    \n",
    "for i, value in enumerate(reg_alpha):\n",
    "    pyplot.plot(reg_lambda, -test_scores[i], label= 'reg_alpha:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'reg_alpha' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "二、测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "xgb = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=219,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        reg_alpha = 1.5,\n",
    "        reg_lambda = 0.1,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "X_test = pd.read_csv(dpath +\"RentListingInquries_FE_test.csv\")\n",
    "y_pred = xgb.fit(X_train, y_train).predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "三、生成测试结果文件  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = pd.DataFrame(pd.Series(data = y_pred, name = 'interest_level'))\n",
    "y.to_csv('submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 74659 entries, 0 to 74658\n",
      "Data columns (total 1 columns):\n",
      "interest_level    74659 non-null int64\n",
      "dtypes: int64(1)\n",
      "memory usage: 583.4 KB\n"
     ]
    }
   ],
   "source": [
    "y.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
